moreminimore-chat/app/services/knowledge_base/import_service.rb

# Imports markdown FAQ content into KnowledgeBaseFaq for an account (OSS self-contained
# chatbot, phase: KB). Accepts:
#   - pasted markdown text (content:) — split into per-heading sections
#   - an uploaded .md / .csv / .xlsx file (file_path: + filename:)
#
# For markdown: each `#`/`##`/`---` section becomes one KnowledgeBaseFaq row, which
# is the retrieval unit (title = heading, content = section body). topic_tags come
# from explicit front-matter tags or fall back to the first heading words.
#
# Returns a Hash: { imported:, updated:, errors: [{ line, message }] }.
class KnowledgeBase::ImportService
  FRONT_MATTER_TAGS = /\A---\s*\ntags:\s*(.+?)\n---\s*\n/im
  # Matches markdown headings (#, ##, ... up to ######). Uses [ # ]{1,6} to avoid
  # any #{ } interpolation ambiguity in the regex literal.
  HEADING = /^[#]{1,6}\s+(.+)$/i

  def self.import(account:, content: nil, file_path: nil, filename: nil)
    new(account: account, content: content, file_path: file_path, filename: filename).import
  end

  def initialize(account:, content: nil, file_path: nil, filename: nil)
    @account = account
    @content = content.to_s
    @file_path = file_path
    @filename = filename
  end

  def import
    sections = @file_path ? sections_from_file : sections_from_text(@content)
    upsert_sections(sections)
  end

  private

  def sections_from_file
    ext = File.extname(@filename.presence || @file_path.to_s).delete('.').downcase
    text =
      case ext
      when 'md'
        File.read(@file_path)
      when 'csv'
        require 'csv'
        # one row per column -> naive title/content pair
        CSV.read(@file_path).map { |r| "#{r[0]}\n\n#{r[1..].join(' ')}" }.join("\n\n")
      when 'xlsx'
        read_xlsx(@file_path)
      else
        File.read(@file_path)
      end
    sections_from_text(text)
  rescue StandardError => e
    Rails.logger.error("[KnowledgeBaseImport] parse failed: #{e.message}")
    [{ error: "could not read file: #{e.message}" }]
  end

  def read_xlsx(path)
    require 'roo'
    sheet = Roo::Spreadsheet.open(path, extension: 'xlsx').sheet(0)
    rows = (1..sheet.last_row).filter_map do |idx|
      r = (1..sheet.last_column).map { |c| sheet.cell(idx, c).to_s }
      "#{r[0]}\n\n#{r[1..].join(' ')}" unless r.all?(&:blank?)
    end
    rows.join("\n\n")
  end

  # Split markdown into per-heading sections; content preceding the first heading is
  # treated as a single section with a derived title. A leading front-matter block
  # (`---\ntags: ...\n---`) is stripped and its tags applied to every section.
  def sections_from_text(text)
    body, tags = extract_front_matter_tags(text.to_s)

    sections = []
    current = { title: nil, body: [] }

    body.strip.split(/\r?\n/).each do |line|
      if (m = line.match(HEADING))
        # Flush the current section (even a title-less intro block) before a heading
        sections << close_section(current, tags)
        current = { title: m[1].strip, body: [] }
      else
        current[:body] << line
      end
    end
    sections << close_section(current, tags)

    sections.compact.reject { |s| s[:content].blank? }
  end

  # Returns [body_without_front_matter, tags_array]
  def extract_front_matter_tags(text)
    if (m = text.match(FRONT_MATTER_TAGS))
      [text.sub(m[0], ''), m[1].split(/[,;\s]+/).map(&:strip).reject(&:blank?)]
    else
      [text, []]
    end
  end

  def close_section(section, tags)
    body = section[:body].join("\n").strip
    return nil if body.blank? && section[:title].blank?

    { title: section[:title].presence || body.lines.first.to_s.strip[0..80], content: body, topic_tags: tags }
  end

  def upsert_sections(sections)
    imported = 0
    updated = 0
    errors = []
    sections.each_with_index do |sec, idx|
      next unless sec.is_a?(Hash) && sec[:content].present?
      next if sec[:content].blank?

      attrs = {
        content: sec[:content],
        topic_tags: sec[:topic_tags] || [],
        source_filename: @filename
      }
      existing = @account.knowledge_base_faqs.find_by(title: sec[:title])
      if existing
        existing.update!(attrs)
        updated += 1
      else
        @account.knowledge_base_faqs.create!(attrs.merge(title: sec[:title]))
        imported += 1
      end
    rescue ActiveRecord::RecordInvalid => e
      errors << { line: idx + 1, message: e.message }
    end
    { imported: imported, updated: updated, errors: errors }
  end
end