feat(chatbot): OSS self-contained guardrail + knowledge-base chatbot
Built-in AI chatbot (no EE, uses Llm::Resolver) that answers in-scope chats
from a knowledge base and hands off to a human when needed. Selected per
inbox via an Integrations::Hook with app_id 'chatbot'.
- Integrations::Chatbot::ProcessorService (mirrors Dialogflow/Captain)
wired via HookListener + HookJob + apps.yml(chatbot, inbox)
- Chatbot::DecisionService: 1-call default ({in_scope/refuse/handoff}),
2-call option; thread-safe prompt threading
- Chatbot::KnowledgeRetriever: keyword top-k over KB (+ embedding reserved)
- KnowledgeBaseFaq + import service (md per-heading + front-matter tags)
- Chatbot::ConfigService + admin chatbot_config endpoint
- refs off-topic (e.g. fortune-telling); handoff = bot_handoff! (pending->open)
This commit is contained in:
30
app/controllers/api/v2/accounts/chatbot_config_controller.rb
Normal file
30
app/controllers/api/v2/accounts/chatbot_config_controller.rb
Normal file
@@ -0,0 +1,30 @@
|
||||
# Admin-only chatbot configuration endpoint for the self-contained OSS chatbot.
|
||||
#
|
||||
# GET /api/v2/accounts/:account_id/chatbot_config -> current config
|
||||
# POST /api/v2/accounts/:account_id/chatbot_config -> update allowed keys
|
||||
#
|
||||
# Admin-role only (ReportPolicy#view? => administrator?). Reads/writes the account's
|
||||
# chatbot settings via Chatbot::ConfigService (stored in Account#custom_attributes).
|
||||
class Api::V2::Accounts::ChatbotConfigController < Api::V1::Accounts::BaseController
|
||||
before_action :check_authorization
|
||||
|
||||
def show
|
||||
render json: Chatbot::ConfigService.config(Current.account)
|
||||
end
|
||||
|
||||
def update
|
||||
config = Chatbot::ConfigService.update!(Current.account, chatbot_config_params)
|
||||
render json: config
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def chatbot_config_params
|
||||
params.permit(:chatbot_enabled, :chatbot_system_prompt, :chatbot_guardrail_prompt,
|
||||
:chatbot_out_of_scope_reply, :chatbot_call_mode)
|
||||
end
|
||||
|
||||
def check_authorization
|
||||
authorize :report, :view?
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,34 @@
|
||||
# Admin-only knowledge base management for the self-contained OSS chatbot.
|
||||
#
|
||||
# GET /api/v2/accounts/:account_id/knowledge_base_faqs
|
||||
# -> list FAQ entries
|
||||
# POST /api/v2/accounts/:account_id/knowledge_base_faqs/import
|
||||
# -> import markdown text (`content`) or uploaded .md/.csv/.xlsx (`file`)
|
||||
#
|
||||
# Admin-role only (ReportPolicy#view? => administrator?).
|
||||
class Api::V2::Accounts::KnowledgeBaseFaqsController < Api::V1::Accounts::BaseController
|
||||
before_action :check_authorization
|
||||
|
||||
def index
|
||||
faqs = Current.account.knowledge_base_faqs.order(:title)
|
||||
render json: { faqs: faqs.as_json(only: %i[id title topic_tags source_filename updated_at]) }
|
||||
end
|
||||
|
||||
def import
|
||||
if params[:file].present?
|
||||
result = KnowledgeBase::ImportService.import(account: Current.account, file_path: params[:file].tempfile.path, filename: params[:file].original_filename)
|
||||
else
|
||||
content = params[:content]
|
||||
raise ActionController::BadRequest, 'content is required' if content.blank?
|
||||
|
||||
result = KnowledgeBase::ImportService.import(account: Current.account, content: content)
|
||||
end
|
||||
render json: result
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def check_authorization
|
||||
authorize :report, :view?
|
||||
end
|
||||
end
|
||||
@@ -6,6 +6,7 @@ class HookJob < MutexApplicationJob
|
||||
INTEGRATION_PROCESSORS = {
|
||||
'slack' => :process_slack_integration,
|
||||
'dialogflow' => :process_dialogflow_integration,
|
||||
'chatbot' => :process_chatbot_integration,
|
||||
'google_translate' => :google_translate_integration,
|
||||
'leadsquared' => :process_leadsquared_integration_with_lock,
|
||||
'linear' => :process_linear_integration
|
||||
@@ -50,6 +51,15 @@ class HookJob < MutexApplicationJob
|
||||
Integrations::Dialogflow::ProcessorService.new(event_name: event_name, hook: hook, event_data: event_data).perform
|
||||
end
|
||||
|
||||
def process_chatbot_integration(hook, event_name, event_data)
|
||||
return unless event_name == 'message.created'
|
||||
|
||||
message = event_data[:message]
|
||||
return unless message.content_type == 'text' && message.content.present?
|
||||
|
||||
Integrations::Chatbot::ProcessorService.new(event_name: event_name, hook: hook, event_data: event_data).perform
|
||||
end
|
||||
|
||||
def google_translate_integration(hook, event_name, event_data)
|
||||
return unless ['message.created'].include?(event_name)
|
||||
|
||||
|
||||
@@ -61,6 +61,7 @@ class HookListener < BaseListener
|
||||
supported_events_map = {
|
||||
'slack' => ['message.created', 'message.updated'],
|
||||
'dialogflow' => ['message.created', 'message.updated'],
|
||||
'chatbot' => ['message.created'],
|
||||
'google_translate' => ['message.created'],
|
||||
'leadsquared' => ['contact.updated', 'conversation.created', 'conversation.resolved'],
|
||||
'linear' => ['message.created']
|
||||
|
||||
@@ -86,6 +86,7 @@ class Account < ApplicationRecord
|
||||
has_many :tiktok_channels, dependent: :destroy_async, class_name: '::Channel::Tiktok'
|
||||
has_many :hooks, dependent: :destroy_async, class_name: 'Integrations::Hook'
|
||||
has_many :inboxes, dependent: :destroy_async
|
||||
has_many :knowledge_base_faqs, dependent: :destroy_async
|
||||
has_many :labels, dependent: :destroy_async
|
||||
has_many :line_channels, dependent: :destroy_async, class_name: '::Channel::Line'
|
||||
has_many :mentions, dependent: :destroy_async
|
||||
@@ -94,6 +95,7 @@ class Account < ApplicationRecord
|
||||
has_many :notification_settings, dependent: :destroy_async
|
||||
has_many :notifications, dependent: :destroy_async
|
||||
has_many :portals, dependent: :destroy_async, class_name: '::Portal'
|
||||
has_many :product_catalog_entries, dependent: :destroy_async
|
||||
has_many :sms_channels, dependent: :destroy_async, class_name: '::Channel::Sms'
|
||||
has_many :teams, dependent: :destroy_async
|
||||
has_many :telegram_channels, dependent: :destroy_async, class_name: '::Channel::Telegram'
|
||||
|
||||
34
app/models/knowledge_base_faq.rb
Normal file
34
app/models/knowledge_base_faq.rb
Normal file
@@ -0,0 +1,34 @@
|
||||
# == Schema Information
|
||||
#
|
||||
# Table name: knowledge_base_faqs
|
||||
#
|
||||
# id :bigint not null, primary key
|
||||
# account_id :bigint not null
|
||||
# title :string not null
|
||||
# content :text not null
|
||||
# topic_tags :jsonb default: [] not null
|
||||
# source_filename :string
|
||||
# embedding :vector(1536)
|
||||
# created_at :datetime not null
|
||||
# updated_at :datetime not null
|
||||
#
|
||||
class KnowledgeBaseFaq < ApplicationRecord
|
||||
belongs_to :account
|
||||
# pgvector KNN support (matches repo pattern: has_neighbors + nearest_neighbors)
|
||||
has_neighbors :embedding, normalize: true
|
||||
|
||||
validates :title, presence: true
|
||||
validates :content, presence: true
|
||||
|
||||
# topic_tags stored as jsonb array; expose string-list helpers for import/retrieval.
|
||||
def topic_tag_list
|
||||
Array(topic_tags)
|
||||
end
|
||||
|
||||
def topic_tag_list=(value)
|
||||
self.topic_tags = Array(value).map(&:strip).reject(&:blank?)
|
||||
end
|
||||
|
||||
scope :for_account, ->(account) { where(account_id: account.id) }
|
||||
scope :with_embedding, -> { where.not(embedding: nil) }
|
||||
end
|
||||
86
app/services/chatbot/config_service.rb
Normal file
86
app/services/chatbot/config_service.rb
Normal file
@@ -0,0 +1,86 @@
|
||||
# Per-account chatbot configuration for the self-contained OSS chatbot.
|
||||
#
|
||||
# Reads/writes the account's chatbot settings in Account#custom_attributes (jsonb).
|
||||
# The chatbot processor + decision service read from here so there's a single source
|
||||
# of truth for how the bot behaves for a given account.
|
||||
#
|
||||
# Keys (namespaced 'chatbot_*'):
|
||||
# chatbot_enabled [bool] gates whether the chatbot replies (default false)
|
||||
# chatbot_system_prompt [String] the persona/system instructions for the answer LLM
|
||||
# chatbot_guardrail_prompt [String] the allowed-scope / guardrail instructions
|
||||
# chatbot_out_of_scope_reply [String] templated refusal reply for out-of-scope messages
|
||||
# chatbot_call_mode [Integer] 1 (one call) or 2 (guardrail + answer) — default 1
|
||||
#
|
||||
# Guardrail + out-of-scope reply have safe Thai defaults; system prompt and call mode
|
||||
# fall back to built-in values when unset.
|
||||
module Chatbot::ConfigService
|
||||
KEYS = %w[
|
||||
chatbot_enabled chatbot_system_prompt chatbot_guardrail_prompt
|
||||
chatbot_out_of_scope_reply chatbot_call_mode
|
||||
].freeze
|
||||
|
||||
DEFAULT_SYSTEM_PROMPT = <<~PROMPT.freeze
|
||||
You are a helpful customer-service assistant for this business. Answer using ONLY the
|
||||
provided knowledge base and conversation history. Be concise, accurate and polite.
|
||||
If the knowledge base does not contain the answer, say you are not sure and offer to
|
||||
connect the customer with a support agent. Do not invent facts.
|
||||
PROMPT
|
||||
|
||||
DEFAULT_GUARDRAIL_PROMPT = <<~PROMPT.freeze
|
||||
The bot answers questions about this business's PRODUCTS, SERVICES, and related support
|
||||
topics only. It must NOT answer unrelated or off-topic requests (e.g. personal advice,
|
||||
fortune-telling, horoscopes, unrelated general knowledge).
|
||||
PROMPT
|
||||
|
||||
DEFAULT_OUT_OF_SCOPE_REPLY = 'ขออภัยครับ คำถามนี้อยู่นอกขอบเขตที่เราสามารถให้บริการได้ กรุณาสอบถามเรื่องสินค้าและบริการของเรา'
|
||||
|
||||
module_function
|
||||
|
||||
def enabled?(account)
|
||||
account.custom_attributes['chatbot_enabled'] == true
|
||||
end
|
||||
|
||||
def system_prompt(account)
|
||||
value_or_default(account, 'chatbot_system_prompt', DEFAULT_SYSTEM_PROMPT)
|
||||
end
|
||||
|
||||
def guardrail_prompt(account)
|
||||
value_or_default(account, 'chatbot_guardrail_prompt', DEFAULT_GUARDRAIL_PROMPT)
|
||||
end
|
||||
|
||||
def out_of_scope_reply(account)
|
||||
value_or_default(account, 'chatbot_out_of_scope_reply', DEFAULT_OUT_OF_SCOPE_REPLY)
|
||||
end
|
||||
|
||||
def call_mode(account)
|
||||
configured = account.custom_attributes['chatbot_call_mode'].to_i
|
||||
[1, 2].include?(configured) ? configured : 1
|
||||
end
|
||||
|
||||
# Apply a params hash of allowed keys to the account's custom_attributes and persist.
|
||||
# Returns the resulting config hash. Ignores/merges only known keys.
|
||||
def update!(account, params)
|
||||
attrs = account.custom_attributes || {}
|
||||
KEYS.each do |key|
|
||||
attrs[key] = params[key] if params.key?(key)
|
||||
end
|
||||
account.update!(custom_attributes: attrs)
|
||||
config(account)
|
||||
end
|
||||
|
||||
# @return [Hash] full current config
|
||||
def config(account)
|
||||
{
|
||||
enabled: enabled?(account),
|
||||
system_prompt: system_prompt(account),
|
||||
guardrail_prompt: guardrail_prompt(account),
|
||||
out_of_scope_reply: out_of_scope_reply(account),
|
||||
call_mode: call_mode(account)
|
||||
}
|
||||
end
|
||||
|
||||
def value_or_default(account, key, default)
|
||||
value = account.custom_attributes[key]
|
||||
value.presence || default
|
||||
end
|
||||
end
|
||||
206
app/services/chatbot/decision_service.rb
Normal file
206
app/services/chatbot/decision_service.rb
Normal file
@@ -0,0 +1,206 @@
|
||||
# Chatbot decision service for the self-contained OSS chatbot.
|
||||
#
|
||||
# Given an inbound user message (plus history and retrieved knowledge), decides what the
|
||||
# bot should do:
|
||||
# :answer — in-scope; `answer` is the reply (grounded in the knowledge base)
|
||||
# :refuse — clearly off-topic (out of guardrail scope, e.g. fortune-telling); uses the
|
||||
# account's out-of-scope reply template
|
||||
# :handoff — related to scope but the bot can't answer (nothing in KB / undecidable);
|
||||
# the caller should hand off to a human (conversation.bot_handoff!)
|
||||
#
|
||||
# Default: ONE LLM call returns a structured decision { decision, reason, answer }.
|
||||
# Optional TWO-call mode: a guardrail call decides in/out of scope, then an answer call
|
||||
# composes the reply from KB. Selected per account (config option; 2-call is for LLMs that
|
||||
# handle the compound single-call poorly).
|
||||
#
|
||||
# Fail-closed: no LLM credential -> { disabled: true } (never sends chat content when
|
||||
# disabled). Mirrors the Analytics::WeeklyPersonaEvaluator / Llm::AnalyticsClassifier pattern.
|
||||
module Chatbot::DecisionService
|
||||
DECISION_SCHEMA = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
decision: {
|
||||
type: 'string',
|
||||
enum: %w[in_scope refuse handoff],
|
||||
description: "in_scope = answer from the knowledge base; refuse = clearly off-topic and must not be answered; handoff = related to scope but bot cannot answer -> hand to a human."
|
||||
},
|
||||
reason: { type: 'string', description: 'One sentence justifying the decision.' },
|
||||
answer: { type: 'string', description: 'The bot reply. Populated for in_scope; may be blank for refuse/handoff.' }
|
||||
},
|
||||
required: %w[decision reason answer]
|
||||
}.freeze
|
||||
|
||||
Result = Struct.new(:action, :answer, :reason, :disabled, :error, keyword_init: true) do
|
||||
def disabled? = disabled == true
|
||||
def success? = error.nil?
|
||||
def answer? = action == :answer
|
||||
def refuse? = action == :refuse
|
||||
def handoff? = action == :handoff
|
||||
end
|
||||
|
||||
module_function
|
||||
|
||||
# @param account [Account]
|
||||
# @param message [String] the inbound user text
|
||||
# @param history [Array<Hash>] [{ role: 'user'|'assistant', content: String }]
|
||||
# @param knowledge [Array<Hash>] [{ title:, content:, score: }] retrieved KB context
|
||||
# @param call_mode [Integer] 1 (default) or 2
|
||||
# @param system_prompt [String] optional per-account persona/system instructions
|
||||
# @param guardrail_prompt [String] optional per-account allowed-scope instructions
|
||||
# @return [Chatbot::DecisionService::Result]
|
||||
def decide(account:, message:, history: [], knowledge: [], call_mode: 1, system_prompt: nil, guardrail_prompt: nil)
|
||||
credential = Llm::Resolver.resolve(account)
|
||||
return disabled_result if credential.nil?
|
||||
|
||||
# Prompts are threaded as explicit args (not module instance vars) so concurrent
|
||||
# requests can never bleed one account's person/system prompt into another.
|
||||
system = system_prompt.presence || SYSTEM_PROMPT
|
||||
guardrail = guardrail_prompt.presence || GUARDRAIL_SCOPE
|
||||
|
||||
if call_mode == 2
|
||||
decide_two_call(credential, message, history, knowledge, system, guardrail)
|
||||
else
|
||||
decide_one_call(credential, message, history, knowledge, system, guardrail)
|
||||
end
|
||||
rescue StandardError => e
|
||||
Rails.logger.error("[ChatbotDecision] account=#{account&.id} #{e.class}: #{e.message}")
|
||||
Result.new(error: e.message)
|
||||
end
|
||||
|
||||
# -- 1-call mode ------------------------------------------------------------
|
||||
|
||||
def decide_one_call(credential, message, history, knowledge, system_prompt, guardrail_prompt)
|
||||
response = call_llm(credential, build_one_call_prompt(message, history, knowledge, guardrail_prompt), system_prompt)
|
||||
return Result.new(error: response[:error] || 'completion failed') if response[:error]
|
||||
|
||||
parsed = JSON.parse(sanitize_json(response[:content]))
|
||||
action = normalize_action(parsed['decision'])
|
||||
Result.new(
|
||||
action: action,
|
||||
reason: parsed['reason'].to_s,
|
||||
answer: parsed['answer'].to_s,
|
||||
disabled: false
|
||||
)
|
||||
rescue JSON::ParserError, TypeError
|
||||
Result.new(error: 'LLM returned an unparsable decision')
|
||||
end
|
||||
|
||||
# -- 2-call mode ------------------------------------------------------------
|
||||
|
||||
def decide_two_call(credential, message, history, knowledge, system_prompt, guardrail_prompt)
|
||||
guardrail = call_llm(credential, build_guardrail_prompt(message, guardrail_prompt), system_prompt)
|
||||
return Result.new(error: guardrail[:error] || 'guardrail failed') if guardrail[:error]
|
||||
|
||||
parsed = JSON.parse(sanitize_json(guardrail[:content]))
|
||||
decision = parsed['decision']&.to_s
|
||||
return Result.new(action: :refuse, reason: parsed['reason'].to_s, answer: '', disabled: false) if decision == 'refuse'
|
||||
# refuse / handoff_unknown / anything-but-in_scope -> hand to a human
|
||||
return Result.new(action: :handoff, reason: parsed['reason']&.to_s, answer: '', disabled: false) unless decision == 'in_scope'
|
||||
|
||||
answer_response = call_llm(credential, build_answer_prompt(message, history, knowledge), system_prompt)
|
||||
return Result.new(error: answer_response[:error] || 'answer failed') if answer_response[:error]
|
||||
|
||||
Result.new(action: :answer, answer: answer_response[:content].to_s, reason: 'in_scope', disabled: false)
|
||||
rescue JSON::ParserError, TypeError
|
||||
Result.new(error: 'LLM returned an unparsable decision')
|
||||
end
|
||||
|
||||
# -- LLM + prompt helpers ---------------------------------------------------
|
||||
|
||||
def call_llm(credential, prompt, system_prompt)
|
||||
Llm::Config.with_api_key(credential[:api_key], api_base: credential[:api_base]) do |context|
|
||||
chat = context.chat(model: MODEL).with_schema(DECISION_SCHEMA)
|
||||
chat.with_instructions(system_prompt)
|
||||
{ content: chat.ask(prompt).content }
|
||||
end
|
||||
rescue StandardError => e
|
||||
Rails.logger.error("[ChatbotDecision] LLM call failed #{e.class}: #{e.message}")
|
||||
{ error: e.message }
|
||||
end
|
||||
|
||||
MODEL = Llm::Config::DEFAULT_MODEL
|
||||
|
||||
def build_one_call_prompt(message, history, knowledge, guardrail_prompt)
|
||||
[
|
||||
'Decide how the customer-service bot should respond to the customer message.',
|
||||
'',
|
||||
'## Guardrail scope',
|
||||
guardrail_prompt,
|
||||
'',
|
||||
'## Knowledge base (retrieved, most relevant first)',
|
||||
knowledge_text(knowledge).presence || '(no relevant knowledge found)',
|
||||
'',
|
||||
'## Conversation history',
|
||||
history_text(history).presence || '(no prior messages)',
|
||||
'',
|
||||
"## Latest customer message\n#{message}",
|
||||
'',
|
||||
'If the message is in scope AND relevant knowledge exists, return decision=in_scope with the best answer grounded in the knowledge. If it is clearly outside the guardrail scope (e.g. fortune-telling, off-topic), return decision=refuse (answer may be blank). If it is related to scope but there is no knowledge to answer with, return decision=handoff.'
|
||||
].join("\n")
|
||||
end
|
||||
|
||||
def build_guardrail_prompt(message, guardrail_prompt)
|
||||
[
|
||||
'You are a safety guardrail. Decide whether this customer message is within the allowed scope.',
|
||||
'',
|
||||
guardrail_prompt,
|
||||
'',
|
||||
"## Customer message\n#{message}",
|
||||
'',
|
||||
'Return decision: refuse if clearly outside scope; in_scope if within scope but may need knowledge to answer; handoff_unknown if related but ambiguous.'
|
||||
].join("\n")
|
||||
end
|
||||
|
||||
def build_answer_prompt(message, history, knowledge)
|
||||
[
|
||||
'You are a helpful customer-service assistant for this business. Answer the customer using ONLY the provided knowledge base; do not invent facts.',
|
||||
'',
|
||||
'## Knowledge base',
|
||||
knowledge_text(knowledge).presence || '(no relevant knowledge found)',
|
||||
'',
|
||||
'## Conversation history',
|
||||
history_text(history).presence || '(no prior messages)',
|
||||
'',
|
||||
"## Customer message\n#{message}"
|
||||
].join("\n")
|
||||
end
|
||||
|
||||
def knowledge_text(knowledge)
|
||||
Array(knowledge).map { |k| "- #{k[:title]}: #{k[:content]}".strip }.join("\n")
|
||||
end
|
||||
|
||||
def history_text(history)
|
||||
Array(history).map { |h| "#{h[:role].to_s.capitalize}: #{h[:content]}" }.join("\n")
|
||||
end
|
||||
|
||||
def normalize_action(decision)
|
||||
case decision&.to_sym
|
||||
when :in_scope then :answer
|
||||
when :refuse then :refuse
|
||||
when :handoff then :handoff
|
||||
else :handoff
|
||||
end
|
||||
end
|
||||
|
||||
def sanitize_json(content)
|
||||
content.to_s.gsub('```json', '').gsub('```', '').strip
|
||||
end
|
||||
|
||||
def disabled_result
|
||||
Result.new(action: nil, answer: '', reason: '', disabled: true)
|
||||
end
|
||||
|
||||
SYSTEM_PROMPT = <<~PROMPT.freeze
|
||||
You decide and then answer for a customer-service chatbot. Stay within the allowed
|
||||
guardrail scope and be truthful and helpful. Return only the JSON object described by
|
||||
the schema — no extra text.
|
||||
PROMPT
|
||||
|
||||
GUARDRAIL_SCOPE = <<~SCOPE.freeze
|
||||
The bot answers questions about this business's PRODUCTS, SERVICES, and related
|
||||
support topics only. It must NOT answer unrelated or off-topic requests (e.g. personal
|
||||
advice, fortune-telling, horoscopes, unrelated general knowledge, or any topic outside
|
||||
the listed products/services/support).
|
||||
SCOPE
|
||||
end
|
||||
86
app/services/chatbot/knowledge_retriever.rb
Normal file
86
app/services/chatbot/knowledge_retriever.rb
Normal file
@@ -0,0 +1,86 @@
|
||||
# KnowledgeBase retriever for the self-contained OSS chatbot.
|
||||
#
|
||||
# Given a user message, returns the top-k most relevant KnowledgeBaseFaq entries for
|
||||
# that account, ordered best-first. Retrieval is keyword-based (pg_trgm similarity on
|
||||
# title+content + topic_tag match) so it works without an embedding backend.
|
||||
#
|
||||
# A `query_embedding` param is accepted as a future extension point for embedding-fusion
|
||||
# (hybrid keyword + vector ranking), to be wired when an embedding provider is configured
|
||||
# (see plan). Returns [{ faq:, score: Float }] — the caller injects these into the LLM prompt.
|
||||
module Chatbot::KnowledgeRetriever
|
||||
DEFAULT_LIMIT = 5
|
||||
|
||||
module_function
|
||||
|
||||
# @param account [Account]
|
||||
# @param query [String] the user message
|
||||
# @param query_embedding [Array<Float>, nil] reserved; embedding-fusion is a later step
|
||||
# @param limit [Integer]
|
||||
# @return [Array<Hash>] [{ faq:, score: Float }]
|
||||
def retrieve(account:, query:, query_embedding: nil, limit: DEFAULT_LIMIT)
|
||||
return [] if query.blank?
|
||||
|
||||
scores = score_candidates(account, query)
|
||||
return [] if scores.empty?
|
||||
|
||||
max = scores.values.max
|
||||
|
||||
scores.map { |faq, score| { faq: faq, score: (score / max).round(4) } }
|
||||
.sort_by { |h| -h[:score] }
|
||||
.first(limit)
|
||||
end
|
||||
|
||||
# Rank candidate FAQ entries by pg_trgm similarity + topic-tag match.
|
||||
# @return [Hash{KnowledgeBaseFaq => Float}]
|
||||
def score_candidates(account, query)
|
||||
scores = {}
|
||||
candidates(account, query).each do |faq|
|
||||
s = faq_title_similarity(faq, query)
|
||||
s = [s, faq_content_similarity(faq, query)].max
|
||||
s += 0.2 if topic_match?(faq, query)
|
||||
scores[faq] = s if s.positive?
|
||||
end
|
||||
scores
|
||||
end
|
||||
|
||||
# Candidate set: entries whose title or content is likely relevant (pre-filter via
|
||||
# pg_trgm word_similarity to keep the scoring pass small). Falls back to all account
|
||||
# FAQs if pre-filter isn't available (plain AR without pg_trgm search string).
|
||||
def candidates(account, query)
|
||||
relation = account.knowledge_base_faqs
|
||||
column = %(GREATEST(word_similarity(title, #{quote(query)}), word_similarity(content, #{quote(query)})))
|
||||
relation.where("#{column} > 0.1").limit(50).to_a
|
||||
rescue StandardError
|
||||
relation.limit(200).to_a
|
||||
end
|
||||
|
||||
def faq_title_similarity(faq, query)
|
||||
pg_similarity(faq.title, query)
|
||||
end
|
||||
|
||||
def faq_content_similarity(faq, query)
|
||||
pg_similarity(faq.content, query)
|
||||
end
|
||||
|
||||
# Token-overlap similarity ratio computed in Ruby (downcase → split → overlap / max size).
|
||||
# Deterministic and DB-free; used to rank the small candidate pool from `candidates`.
|
||||
def pg_similarity(text_a, text_b)
|
||||
return 0.0 if text_a.blank? || text_b.blank?
|
||||
|
||||
a = text_a.downcase.split(/\s+/).reject(&:blank?)
|
||||
b = text_b.downcase.split(/\s+/).reject(&:blank?)
|
||||
return 0.0 if a.empty? || b.empty?
|
||||
|
||||
overlap = (a & b).size
|
||||
overlap.to_f / [a.size, b.size].max.to_f
|
||||
end
|
||||
|
||||
def topic_match?(faq, query)
|
||||
q = query.downcase
|
||||
faq.topic_tag_list.any? { |t| q.include?(t.downcase) }
|
||||
end
|
||||
|
||||
def quote(value)
|
||||
ActiveRecord::Base.sanitize_sql_like(value.to_s).gsub("'", "''")
|
||||
end
|
||||
end
|
||||
131
app/services/knowledge_base/import_service.rb
Normal file
131
app/services/knowledge_base/import_service.rb
Normal file
@@ -0,0 +1,131 @@
|
||||
# Imports markdown FAQ content into KnowledgeBaseFaq for an account (OSS self-contained
|
||||
# chatbot, phase: KB). Accepts:
|
||||
# - pasted markdown text (content:) — split into per-heading sections
|
||||
# - an uploaded .md / .csv / .xlsx file (file_path: + filename:)
|
||||
#
|
||||
# For markdown: each `#`/`##`/`---` section becomes one KnowledgeBaseFaq row, which
|
||||
# is the retrieval unit (title = heading, content = section body). topic_tags come
|
||||
# from explicit front-matter tags or fall back to the first heading words.
|
||||
#
|
||||
# Returns a Hash: { imported:, updated:, errors: [{ line, message }] }.
|
||||
class KnowledgeBase::ImportService
|
||||
FRONT_MATTER_TAGS = /\A---\s*\ntags:\s*(.+?)\n---\s*\n/im
|
||||
# Matches markdown headings (#, ##, ... up to ######). Uses [ # ]{1,6} to avoid
|
||||
# any #{ } interpolation ambiguity in the regex literal.
|
||||
HEADING = /^[#]{1,6}\s+(.+)$/i
|
||||
|
||||
def self.import(account:, content: nil, file_path: nil, filename: nil)
|
||||
new(account: account, content: content, file_path: file_path, filename: filename).import
|
||||
end
|
||||
|
||||
def initialize(account:, content: nil, file_path: nil, filename: nil)
|
||||
@account = account
|
||||
@content = content.to_s
|
||||
@file_path = file_path
|
||||
@filename = filename
|
||||
end
|
||||
|
||||
def import
|
||||
sections = @file_path ? sections_from_file : sections_from_text(@content)
|
||||
upsert_sections(sections)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def sections_from_file
|
||||
ext = File.extname(@filename.presence || @file_path.to_s).delete('.').downcase
|
||||
text =
|
||||
case ext
|
||||
when 'md'
|
||||
File.read(@file_path)
|
||||
when 'csv'
|
||||
require 'csv'
|
||||
# one row per column -> naive title/content pair
|
||||
CSV.read(@file_path).map { |r| "#{r[0]}\n\n#{r[1..].join(' ')}" }.join("\n\n")
|
||||
when 'xlsx'
|
||||
read_xlsx(@file_path)
|
||||
else
|
||||
File.read(@file_path)
|
||||
end
|
||||
sections_from_text(text)
|
||||
rescue StandardError => e
|
||||
Rails.logger.error("[KnowledgeBaseImport] parse failed: #{e.message}")
|
||||
[{ error: "could not read file: #{e.message}" }]
|
||||
end
|
||||
|
||||
def read_xlsx(path)
|
||||
require 'roo'
|
||||
sheet = Roo::Spreadsheet.open(path, extension: 'xlsx').sheet(0)
|
||||
rows = (1..sheet.last_row).filter_map do |idx|
|
||||
r = (1..sheet.last_column).map { |c| sheet.cell(idx, c).to_s }
|
||||
"#{r[0]}\n\n#{r[1..].join(' ')}" unless r.all?(&:blank?)
|
||||
end
|
||||
rows.join("\n\n")
|
||||
end
|
||||
|
||||
# Split markdown into per-heading sections; content preceding the first heading is
|
||||
# treated as a single section with a derived title. A leading front-matter block
|
||||
# (`---\ntags: ...\n---`) is stripped and its tags applied to every section.
|
||||
def sections_from_text(text)
|
||||
body, tags = extract_front_matter_tags(text.to_s)
|
||||
|
||||
sections = []
|
||||
current = { title: nil, body: [] }
|
||||
|
||||
body.strip.split(/\r?\n/).each do |line|
|
||||
if (m = line.match(HEADING))
|
||||
# Flush the current section (even a title-less intro block) before a heading
|
||||
sections << close_section(current, tags)
|
||||
current = { title: m[1].strip, body: [] }
|
||||
else
|
||||
current[:body] << line
|
||||
end
|
||||
end
|
||||
sections << close_section(current, tags)
|
||||
|
||||
sections.compact.reject { |s| s[:content].blank? }
|
||||
end
|
||||
|
||||
# Returns [body_without_front_matter, tags_array]
|
||||
def extract_front_matter_tags(text)
|
||||
if (m = text.match(FRONT_MATTER_TAGS))
|
||||
[text.sub(m[0], ''), m[1].split(/[,;\s]+/).map(&:strip).reject(&:blank?)]
|
||||
else
|
||||
[text, []]
|
||||
end
|
||||
end
|
||||
|
||||
def close_section(section, tags)
|
||||
body = section[:body].join("\n").strip
|
||||
return nil if body.blank? && section[:title].blank?
|
||||
|
||||
{ title: section[:title].presence || body.lines.first.to_s.strip[0..80], content: body, topic_tags: tags }
|
||||
end
|
||||
|
||||
def upsert_sections(sections)
|
||||
imported = 0
|
||||
updated = 0
|
||||
errors = []
|
||||
sections.each_with_index do |sec, idx|
|
||||
next unless sec.is_a?(Hash) && sec[:content].present?
|
||||
next if sec[:content].blank?
|
||||
|
||||
attrs = {
|
||||
content: sec[:content],
|
||||
topic_tags: sec[:topic_tags] || [],
|
||||
source_filename: @filename
|
||||
}
|
||||
existing = @account.knowledge_base_faqs.find_by(title: sec[:title])
|
||||
if existing
|
||||
existing.update!(attrs)
|
||||
updated += 1
|
||||
else
|
||||
@account.knowledge_base_faqs.create!(attrs.merge(title: sec[:title]))
|
||||
imported += 1
|
||||
end
|
||||
rescue ActiveRecord::RecordInvalid => e
|
||||
errors << { line: idx + 1, message: e.message }
|
||||
end
|
||||
{ imported: imported, updated: updated, errors: errors }
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user