moreminimore-chat/app/services/analytics/weekly_persona_evaluator.rb

# Weekly persona evaluation for the self-improving chatbot (phase 3).
#
# Summarizes the last 7 days of immutable daily metrics (Analytics::ReportService /
# ConversationDailyMetric + CustomerDailyMetric via Analytics::ReportService) and asks the
# LLM (via the Llm::Resolver cascade) to recommend persona/system-prompt improvements.
#
# Output is the SUMMARY ONLY (human-readable recommendation) — the full system prompt is
# intentionally NOT produced/revealed here; it is gated behind the admin approval flow.
#
# Like Llm::AnalyticsClassifier, this is a pure evaluator: it CLASSIFIES/SUMMARIZES and
# returns a Result; persisting the recommendation is the caller's responsibility
# (the weekly job / approval flow). Fail-closed: no LLM credential -> { disabled: true },
# never sends conversation content when disabled.
module Analytics::WeeklyPersonaEvaluator
  SCHEMA = {
    type: 'object',
    additionalProperties: false,
    properties: {
      summary: {
        type: 'string',
        description: 'A concise human-readable summary of the week: top topics, sales wins/losses, and any notable trends.'
      },
      recommendations: {
        type: 'array',
        items: { type: 'string' },
        description: 'Concrete, actionable recommendations to improve the chatbot persona/behavior next week.'
      }
    },
    required: %w[summary recommendations]
  }.freeze

  Result = Struct.new(:summary, :recommendations, :disabled, :error, keyword_init: true) do
    def disabled?
      disabled == true
    end

    def success?
      error.nil?
    end
  end

  WINDOW_DAYS = 7

  module_function

  # @param account [Account]
  # @param report [Hash] output of Analytics::ReportService.build (or built here if nil)
  # @return [Analytics::WeeklyPersonaEvaluator::Result]
  def evaluate(account:, report: nil)
    credential = Llm::Resolver.resolve(account)
    return disabled_result if credential.nil?

    report ||= Analytics::ReportService.build(account: account, since: WINDOW_DAYS.days.ago.to_date, until_date: Date.today)
    response = call_llm(credential, build_prompt(report))
    build_result(response)
  rescue StandardError => e
    Rails.logger.error("[WeeklyPersonaEvaluator] account=#{account&.id} #{e.class}: #{e.message}")
    Result.new(error: e.message)
  end

  # -- result helpers ---------------------------------------------------------

  def disabled_result
    Result.new(summary: nil, recommendations: [], disabled: true)
  end

  def build_result(response)
    return Result.new(error: response[:error] || 'evaluation failed') if response[:error]

    parsed = JSON.parse(sanitize_json(response[:content]))
    Result.new(
      summary: parsed['summary'],
      recommendations: Array(parsed['recommendations']),
      disabled: false
    )
  rescue JSON::ParserError, TypeError
    Result.new(error: 'LLM returned an unparsable evaluation')
  end

  # -- LLM call ---------------------------------------------------------------

  def call_llm(credential, prompt)
    Llm::Config.with_api_key(credential[:api_key], api_base: credential[:api_base]) do |context|
      chat = context.chat(model: MODEL).with_schema(SCHEMA)
      chat.with_instructions(SYSTEM_PROMPT)
      { content: chat.ask(prompt).content }
    end
  rescue StandardError => e
    Rails.logger.error("[WeeklyPersonaEvaluator] LLM call failed #{e.class}: #{e.message}")
    { error: e.message }
  end

  MODEL = Llm::Config::DEFAULT_MODEL

  # -- prompt construction ----------------------------------------------------

  def build_prompt(report)
    summary = report[:summary].to_h
    [
      'Here is the past week of customer-service analytics for the account:',
      '',
      "Conversations: #{summary[:conversation_count]}",
      "Messages: #{summary[:message_count]}",
      "Resolved: #{summary[:resolved_count]}",
      "Unresolved: #{summary[:unresolved_count]}",
      "Deal outcomes: #{summary[:deal_outcomes].inspect}",
      "Top tags: #{summary[:top_tags].inspect}",
      '',
      'Based on this, recommend persona / behavior improvements for the chatbot.'
    ].join("\n")
  end

  def sanitize_json(content)
    content.to_s.gsub('```json', '').gsub('```', '').strip
  end

  SYSTEM_PROMPT = <<~PROMPT.freeze
    You are a customer-service improvement analyst. Given a week of aggregate metrics,
    write a concise summary and 2-5 concrete, actionable recommendations to improve the
    chatbot's persona and behavior. Keep recommendations specific and grounded in the data.
    Return only the JSON object described by the schema — no extra text.
  PROMPT
end