From 0a2293f9219af5dcc54f8bc0a8f7849d100da36b Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 11 Aug 2026 14:57:23 +0530 Subject: [PATCH] feat: record Captain conversation outcome episodes from lifecycle events [CW-7792] (#15316) Records Captain conversation outcomes at episode grain so reporting can distinguish initial demand from reopened conversations and measure replies, handoffs, resolutions, human follow-up, and CSAT. Eligibility creates the episode at demand time. Message-derived fields are snapshotted from persisted messages at handoff or resolution, keeping terminal analytics accurate without writing outcomes for every message. Outcome tracking remains reporting-only and fail-open. Builds on the episode-grain schema from #15315. ## Closes - https://linear.app/chatwoot/issue/CW-7792 ## How to test 1. Enable `captain_integration_v2` and connect a Captain assistant to an inbox. 2. Send an inbound customer message and confirm an initial outcome episode is created at the message timestamp. 3. Let Captain reply and then resolve or hand off the conversation. Confirm the episode records Captain reply counts and timestamps, the outcome timestamp, and the handoff category where applicable. 4. Reply after resolution and confirm a `reopen` episode is created while preserving the previous episode. 5. Resolve the reopened conversation and submit CSAT. Confirm the response is attributed to the episode that issued the survey. ## What changed - Creates the initial episode from demand-level eligibility and appends a new episode when a resolved conversation reopens. - Snapshots Captain replies and the first qualifying human reply from persisted messages at handoff and resolution. - Attributes asynchronous resolution events using the episode active at the event timestamp. - Records later CSAT responses using the survey message timestamp. - Keeps boundary writes transactional and fail-open without retries, advisory locks, late-boundary repair, or handoff self-healing. - Adds schema-constrained handoff reason categories, including lifecycle coverage for incomplete V2 tool fallback handoffs. Open, non-terminal episodes may retain empty or stale message-derived fields until handoff or resolution. --- app/dispatchers/sync_dispatcher.rb | 2 + .../dispatchers/enterprise/sync_dispatcher.rb | 7 + .../conversation/response_builder_job.rb | 14 +- .../conversation/v2_lifecycle_events.rb | 4 +- .../conversation_outcome_event_listener.rb | 25 +++ enterprise/app/listeners/captain_listener.rb | 18 ++ .../response_scheduler_service.rb | 12 -- .../services/captain/conversation_events.rb | 11 +- .../captain/conversation_outcome_tracker.rb | 129 ++++++++++++ .../hook_execution_service.rb | 43 ++-- enterprise/lib/captain/tools/handoff_tool.rb | 48 ++++- lib/events/types.rb | 1 - .../conversation/response_builder_job_spec.rb | 2 + .../lib/captain/tools/handoff_tool_spec.rb | 63 ++++-- .../listeners/captain_listener_spec.rb | 71 ++++++- .../captain/conversation_events_spec.rb | 16 +- .../conversation_outcome_tracker_spec.rb | 195 ++++++++++++++++++ .../hook_execution_service_spec.rb | 51 +++-- 18 files changed, 608 insertions(+), 104 deletions(-) create mode 100644 enterprise/app/dispatchers/enterprise/sync_dispatcher.rb create mode 100644 enterprise/app/listeners/captain/conversation_outcome_event_listener.rb create mode 100644 enterprise/app/services/captain/conversation_outcome_tracker.rb create mode 100644 spec/enterprise/services/captain/conversation_outcome_tracker_spec.rb diff --git a/app/dispatchers/sync_dispatcher.rb b/app/dispatchers/sync_dispatcher.rb index 509a42727..837e8af7c 100644 --- a/app/dispatchers/sync_dispatcher.rb +++ b/app/dispatchers/sync_dispatcher.rb @@ -8,3 +8,5 @@ class SyncDispatcher < BaseDispatcher [ActionCableListener.instance, AgentBotListener.instance] end end + +SyncDispatcher.prepend_mod_with('SyncDispatcher') diff --git a/enterprise/app/dispatchers/enterprise/sync_dispatcher.rb b/enterprise/app/dispatchers/enterprise/sync_dispatcher.rb new file mode 100644 index 000000000..3e32eb6fc --- /dev/null +++ b/enterprise/app/dispatchers/enterprise/sync_dispatcher.rb @@ -0,0 +1,7 @@ +module Enterprise::SyncDispatcher + def listeners + super + [ + Captain::ConversationOutcomeEventListener.instance + ] + end +end diff --git a/enterprise/app/jobs/captain/conversation/response_builder_job.rb b/enterprise/app/jobs/captain/conversation/response_builder_job.rb index 45b6d9ad2..66f3259b0 100644 --- a/enterprise/app/jobs/captain/conversation/response_builder_job.rb +++ b/enterprise/app/jobs/captain/conversation/response_builder_job.rb @@ -88,7 +88,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob return unless conversation_pending? process_v1_handoff - record_v2_failure_handoff if v2_generation_errored? + record_v2_failure_handoff(source: Captain::ConversationEvents::Sources::GENERATION_FAILURE) if v2_generation_errored? end def process_standard_response @@ -112,11 +112,8 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob if captain_v2_enabled? return unless v2_handoff_tool_completed? || conversation_pending? - # Known gap, accepted for now: the fallback V1 handoff (tool fired but never - # completed) emits no captain.conversation.handed_off event — the tool emits - # only after a successful bot_handoff!. If outcome data ever needs it, emit - # here with a distinct source such as 'tool_fallback'. v2_handoff_tool_completed? ? process_v2_handoff : process_v1_handoff + record_v2_failure_handoff(source: Captain::ConversationEvents::Sources::TOOL) unless v2_handoff_tool_completed? else conversation_pending? ? process_v1_handoff : process_v2_handoff end @@ -136,10 +133,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob @response['response'] == 'conversation_handoff' end - def v2_handoff_tool_fired? - @response['handoff_tool_called'] - end - + def v2_handoff_tool_fired? = @response['handoff_tool_called'] def v2_handoff_tool_completed? = @v2_handoff_tool_completed == true def process_v1_handoff @@ -201,7 +195,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob return if captain_v2_enabled? && newer_customer_message_arrived? process_v1_handoff - record_v2_failure_handoff if captain_v2_enabled? + record_v2_failure_handoff(source: Captain::ConversationEvents::Sources::GENERATION_FAILURE) if captain_v2_enabled? end def log_error(error) diff --git a/enterprise/app/jobs/captain/conversation/v2_lifecycle_events.rb b/enterprise/app/jobs/captain/conversation/v2_lifecycle_events.rb index 37501e9ff..dc1738946 100644 --- a/enterprise/app/jobs/captain/conversation/v2_lifecycle_events.rb +++ b/enterprise/app/jobs/captain/conversation/v2_lifecycle_events.rb @@ -18,11 +18,11 @@ module Captain::Conversation::V2LifecycleEvents Captain::ConversationEvents.response_failed(conversation: @conversation, assistant: @assistant, reason: reason, at: Time.current) end - def record_v2_failure_handoff + def record_v2_failure_handoff(source:) Captain::ConversationEvents.handed_off( conversation: @conversation, assistant: @assistant, - source: Captain::ConversationEvents::Sources::GENERATION_FAILURE, + source: source, reason_category: :tool_failure, at: Time.current ) diff --git a/enterprise/app/listeners/captain/conversation_outcome_event_listener.rb b/enterprise/app/listeners/captain/conversation_outcome_event_listener.rb new file mode 100644 index 000000000..fa5848920 --- /dev/null +++ b/enterprise/app/listeners/captain/conversation_outcome_event_listener.rb @@ -0,0 +1,25 @@ +class Captain::ConversationOutcomeEventListener < BaseListener + def message_created(event) + message = event.data[:message] + Captain::ConversationOutcomeTracker.new(conversation: message.conversation).record_human_reply(message: message) + end + + def captain_conversation_handed_off(event) + tracker(event).record_handoff(at: event.timestamp, reason_category: event.data[:reason_category]) + end + + def conversation_updated(event) + return unless event.data[:changed_attributes]&.dig('status', 0) == 'resolved' + + tracker(event).record_reopen(at: event.timestamp) + end + + private + + def tracker(event) + Captain::ConversationOutcomeTracker.new( + conversation: event.data[:conversation], + assistant: event.data[:assistant] + ) + end +end diff --git a/enterprise/app/listeners/captain_listener.rb b/enterprise/app/listeners/captain_listener.rb index cd6d8fd1f..01b2605d3 100644 --- a/enterprise/app/listeners/captain_listener.rb +++ b/enterprise/app/listeners/captain_listener.rb @@ -1,8 +1,20 @@ class CaptainListener < BaseListener include ::Events::Types + def message_updated(event) + message = event.data[:message] + return unless message.input_csat? + + response = CsatSurveyResponse.find_by(message: message) + return unless response + + tracker(message.conversation).record_csat(response: response) + end + def conversation_resolved(event) conversation = extract_conversation_and_account(event)[0] + tracker(conversation).record_resolution(at: event.timestamp) + assistant = conversation.inbox.captain_assistant return unless conversation.inbox.captain_active? @@ -10,4 +22,10 @@ class CaptainListener < BaseListener Captain::Llm::ContactNotesService.new(assistant, conversation).generate_and_update_notes if assistant.config['feature_memory'].present? Captain::Llm::ConversationFaqJob.perform_later(conversation, assistant) if assistant.config['feature_faq'].present? end + + private + + def tracker(conversation) + Captain::ConversationOutcomeTracker.new(conversation: conversation) + end end diff --git a/enterprise/app/services/captain/conversation/response_scheduler_service.rb b/enterprise/app/services/captain/conversation/response_scheduler_service.rb index 0e8f6c508..692e10501 100644 --- a/enterprise/app/services/captain/conversation/response_scheduler_service.rb +++ b/enterprise/app/services/captain/conversation/response_scheduler_service.rb @@ -8,8 +8,6 @@ class Captain::Conversation::ResponseSchedulerService end def perform - track_captain_engagement - wait_time = attachment_wait_time return Captain::Conversation::ResponseBuilderJob.perform_later(*job_args) if wait_time.zero? @@ -24,16 +22,6 @@ class Captain::Conversation::ResponseSchedulerService args end - def track_captain_engagement - return unless captain_v2_enabled? - - Captain::ConversationEvents.engaged( - conversation: @conversation, - assistant: @assistant, - at: @message.created_at - ) - end - def captain_v2_enabled? @conversation.account.feature_enabled?('captain_integration_v2') end diff --git a/enterprise/app/services/captain/conversation_events.rb b/enterprise/app/services/captain/conversation_events.rb index 964b2d365..9ad8c3793 100644 --- a/enterprise/app/services/captain/conversation_events.rb +++ b/enterprise/app/services/captain/conversation_events.rb @@ -8,16 +8,7 @@ class Captain::ConversationEvents end class << self - def engaged(conversation:, assistant:, at:) - dispatch( - Events::Types::CAPTAIN_CONVERSATION_ENGAGED, - at: at, - conversation: conversation, - assistant: assistant - ) - end - - def handed_off(conversation:, assistant:, source:, at:, reason_category: nil) + def handed_off(conversation:, assistant:, source:, reason_category:, at:) dispatch( Events::Types::CAPTAIN_CONVERSATION_HANDED_OFF, at: at, diff --git a/enterprise/app/services/captain/conversation_outcome_tracker.rb b/enterprise/app/services/captain/conversation_outcome_tracker.rb new file mode 100644 index 000000000..3cae8201d --- /dev/null +++ b/enterprise/app/services/captain/conversation_outcome_tracker.rb @@ -0,0 +1,129 @@ +class Captain::ConversationOutcomeTracker + pattr_initialize [:conversation!, :assistant] + + def record_eligibility(at:) + safely_track(:eligibility, at: at) do + episodes.trigger_initial.take || create_initial_episode(at) + end + end + + def record_reopen(at:) + safely_track(:reopen, at: at) do + next if episodes.none? + + ApplicationRecord.transaction do + predecessor = episodes.find_by!(ended_at: nil) + predecessor.update!(ended_at: at) + ConversationOutcome.create!( + account: account, + assistant: conversation.inbox.captain_assistant || predecessor.assistant, + conversation: conversation, + inbox: conversation.inbox, + episode_trigger: 'reopen', + started_at: at + ) + end + end + end + + def record_handoff(at:, reason_category:) + snapshot_episode(:handoff, at: at) do |episode| + episode.handoff_at = at + episode.handoff_reason_category = reason_category + end + end + + def record_resolution(at:) + snapshot_episode(:resolution, at: at) do |episode| + episode.resolved_at = at + end + end + + def record_human_reply(message:) + return unless public_human_reply?(message) + + safely_track(:human_reply, at: message.created_at) do + episode = attributed_episode(message.created_at) + next unless episode + next episode if episode.first_human_reply_at.present? + + episode.update!(first_human_reply_at: message.created_at) + episode + end + end + + def record_csat(response:) + safely_track(:csat, at: response.message.created_at) do + episode = attributed_episode(response.message.created_at) + next unless episode + + episode.update!(csat_rating: response.rating, csat_received_at: response.created_at) + episode + end + end + + private + + def account + conversation.account + end + + def episodes + ConversationOutcome.where(account_id: conversation.account_id, conversation_id: conversation.id) + end + + def create_initial_episode(at) + ConversationOutcome.create!( + account: account, + assistant: assistant, + conversation: conversation, + inbox: conversation.inbox, + episode_trigger: 'initial', + started_at: at + ) + end + + def snapshot_episode(action, at:) + safely_track(action, at: at) do + episode = attributed_episode(at) + next unless episode + + yield episode + snapshot_message_facts(episode, through: at) + episode.save! if episode.changed? + episode + end + end + + def attributed_episode(at) + episodes.covering(at).chronological.last + end + + def snapshot_message_facts(episode, through:) + messages = conversation.messages.where(created_at: episode.started_at..through) + replies = messages.where(sender_type: 'Captain::Assistant', message_type: :outgoing, private: false) + + episode.captain_reply_count = replies.count + episode.first_captain_reply_at = replies.minimum(:created_at) + episode.last_captain_reply_at = replies.maximum(:created_at) + end + + def public_human_reply?(message) + return false unless message.outgoing? && !message.private? + return false if message.content_attributes['automation_rule_id'].present? + return false if message.additional_attributes['campaign_id'].present? + + message.sender.is_a?(User) || message.content_attributes['external_echo'].present? + end + + def safely_track(action, at:) + yield + rescue StandardError => e + ChatwootExceptionTracker.new(e, account: account).capture_exception + Rails.logger.error( + "[CAPTAIN][ConversationOutcomeTracker] Failed to record #{action} for conversation=#{conversation.display_id} " \ + "at=#{at.iso8601(6)}: #{e.message}" + ) + nil + end +end diff --git a/enterprise/app/services/enterprise/message_templates/hook_execution_service.rb b/enterprise/app/services/enterprise/message_templates/hook_execution_service.rb index 22997b0ca..6f289339c 100644 --- a/enterprise/app/services/enterprise/message_templates/hook_execution_service.rb +++ b/enterprise/app/services/enterprise/message_templates/hook_execution_service.rb @@ -1,7 +1,15 @@ module Enterprise::MessageTemplates::HookExecutionService def trigger_templates super - return unless should_process_captain_response? + return unless captain_conversation_message? + + # Eligibility is demand-level: every inbound customer message in a + # Captain-connected inbox counts, including conversations a human grabbed + # first or that arrive while the account is over its usage limit — + # otherwise the coverage denominator only ever contains conversations + # Captain was already about to answer. + track_captain_eligibility + return unless conversation.pending? return perform_handoff unless inbox.captain_active? Captain::Conversation::ResponseSchedulerService.new(message: message).perform @@ -27,19 +35,20 @@ module Enterprise::MessageTemplates::HookExecutionService private - def captain_v2_enabled? - conversation.account.feature_enabled?('captain_integration_v2') + def track_captain_eligibility + return unless conversation.account.feature_enabled?('captain_integration_v2') + + Captain::ConversationOutcomeTracker.new( + conversation: conversation, + assistant: inbox.captain_assistant + ).record_eligibility(at: message.created_at) end - def should_process_captain_response? - # Audience and schedule are decided when Captain first takes or reopens a conversation. - # Do not re-evaluate an existing pending conversation for each new message. - conversation.pending? && message.captain_response_triggering? && captain_assistant_configured? && !inbox.external_bot_active? + def captain_conversation_message? + message.captain_response_triggering? && captain_assistant_configured? && !inbox.external_bot_active? end def perform_handoff - return unless conversation.pending? - Rails.logger.info("Captain limit exceeded, performing handoff mid-conversation for conversation: #{conversation.id}") conversation.messages.create!( message_type: :outgoing, @@ -48,15 +57,13 @@ module Enterprise::MessageTemplates::HookExecutionService content: 'Transferring to another agent for further assistance.' ) conversation.bot_handoff! - if captain_v2_enabled? - Captain::ConversationEvents.handed_off( - conversation: conversation, - assistant: inbox.captain_assistant, - source: Captain::ConversationEvents::Sources::USAGE_LIMIT, - reason_category: :usage_limit, - at: Time.current - ) - end + Captain::ConversationEvents.handed_off( + conversation: conversation, + assistant: inbox.captain_assistant, + source: Captain::ConversationEvents::Sources::USAGE_LIMIT, + reason_category: :usage_limit, + at: Time.current + ) send_out_of_office_message_after_handoff end diff --git a/enterprise/lib/captain/tools/handoff_tool.rb b/enterprise/lib/captain/tools/handoff_tool.rb index e1d8a11c8..deb8b0ccf 100644 --- a/enterprise/lib/captain/tools/handoff_tool.rb +++ b/enterprise/lib/captain/tools/handoff_tool.rb @@ -1,8 +1,24 @@ class Captain::Tools::HandoffTool < Captain::Tools::BasePublicTool - description 'Hand off the conversation to a human agent when unable to assist further' - param :reason, type: 'string', desc: 'The reason why handoff is needed (optional)', required: false + # LLM-selectable reasons are a subset of the outcome enum. System lifecycle + # paths emit the remaining categories, such as usage limits and pending clarification. + REASON_CATEGORIES = %w[customer_request missing_knowledge unsupported_request policy_restriction tool_failure].freeze - def perform(tool_context, reason: nil) + description 'Hand off the conversation to a human agent when unable to assist further' + params do + string :reason, description: 'The reason why handoff is needed (optional)', required: false + string :reason_category, enum: REASON_CATEGORIES, description: 'Reporting category for why the handoff is needed' + end + + # Agents::ToolWrapper reads `tool.class.params`, while ruby_llm treats a + # no-argument call as a schema reset. Keep the compatibility fix local to the + # only tool that uses ruby_llm's block schema DSL. + def self.params(schema = nil, &) + return params_schema_definition if schema.nil? && !block_given? + + super + end + + def perform(tool_context, reason: nil, reason_category: nil) conversation = find_conversation(tool_context.state) return 'Conversation not found' unless conversation @@ -13,7 +29,7 @@ class Captain::Tools::HandoffTool < Captain::Tools::BasePublicTool }) # Use existing handoff mechanism from ResponseBuilderJob - handoff_result = trigger_handoff(tool_context, conversation, reason) + handoff_result = trigger_handoff(tool_context, conversation, reason, reason_category) return 'Handoff skipped because a newer customer message arrived' if handoff_result == :stale return 'Handoff skipped because the conversation changed' unless handoff_result == :completed @@ -25,8 +41,8 @@ class Captain::Tools::HandoffTool < Captain::Tools::BasePublicTool private - def trigger_handoff(tool_context, conversation, reason) - return trigger_legacy_handoff(tool_context, conversation, reason) unless captain_v2_enabled? + def trigger_handoff(tool_context, conversation, reason, reason_category) + return trigger_legacy_handoff(tool_context, conversation, reason, reason_category) unless captain_v2_enabled? note = nil handoff_result = conversation.with_lock do @@ -54,28 +70,38 @@ class Captain::Tools::HandoffTool < Captain::Tools::BasePublicTool tool_context.state[:captain_v2_handoff_tool_completed] = true # Queue the event after the state change commits so notification jobs always see the open conversation. conversation.dispatch_bot_handoff_event - emit_tool_handoff_event(conversation) + emit_tool_handoff_event(conversation, reason_category) # Send out of office message if applicable (since template messages were suppressed while Captain was handling) send_out_of_office_message_if_applicable(conversation) :completed end - def trigger_legacy_handoff(tool_context, conversation, reason) + def trigger_legacy_handoff(tool_context, conversation, reason, reason_category) note = conversation.messages.create!( message_type: :outgoing, private: true, sender: @assistant, account: conversation.account, inbox: conversation.inbox, content: reason ) record_handoff_note(tool_context, note) if reason.present? conversation.bot_handoff! - emit_tool_handoff_event(conversation) + emit_tool_handoff_event(conversation, reason_category) send_out_of_office_message_if_applicable(conversation) :completed end - def emit_tool_handoff_event(conversation) + def emit_tool_handoff_event(conversation, reason_category) Captain::ConversationEvents.handed_off(conversation: conversation, assistant: @assistant, - source: Captain::ConversationEvents::Sources::TOOL, at: Time.current) + source: Captain::ConversationEvents::Sources::TOOL, + reason_category: normalize_reason_category(reason_category), + at: Time.current) + end + + # Tool execution does not enforce the schema enum, and an unknown category + # would fail the outcome's validated enum after the handoff already happened. + # Record those handoffs as unclassified instead. + def normalize_reason_category(reason_category) + category = reason_category.to_s + category if REASON_CATEGORIES.include?(category) end def record_handoff_note(tool_context, note) diff --git a/lib/events/types.rb b/lib/events/types.rb index 653739826..83f3d0d3c 100644 --- a/lib/events/types.rb +++ b/lib/events/types.rb @@ -22,7 +22,6 @@ module Events::Types # FIXME: deprecate the opened and resolved events in future in favor of status changed event. CONVERSATION_OPENED = 'conversation.opened' CONVERSATION_RESOLVED = 'conversation.resolved' - CAPTAIN_CONVERSATION_ENGAGED = 'captain.conversation.engaged' CAPTAIN_CONVERSATION_HANDED_OFF = 'captain.conversation.handed_off' CAPTAIN_CONVERSATION_RESOLVED = 'captain.conversation.resolved' CAPTAIN_RESPONSE_COMPLETED = 'captain.response.completed' diff --git a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb index 1d7fe151f..0033e3619 100644 --- a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb +++ b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb @@ -817,6 +817,8 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do 'response' => 'I tried to hand off', 'handoff_tool_called' => true }) + expect(Captain::ConversationEvents).to receive(:handed_off) + .with(conversation: conversation, assistant: assistant, source: 'tool', reason_category: :tool_failure, at: kind_of(Time)) described_class.perform_now(conversation, assistant) diff --git a/spec/enterprise/lib/captain/tools/handoff_tool_spec.rb b/spec/enterprise/lib/captain/tools/handoff_tool_spec.rb index d928e7d8f..51f7ffdda 100644 --- a/spec/enterprise/lib/captain/tools/handoff_tool_spec.rb +++ b/spec/enterprise/lib/captain/tools/handoff_tool_spec.rb @@ -16,13 +16,23 @@ RSpec.describe Captain::Tools::HandoffTool, type: :model do end end - describe '#parameters' do - it 'returns the correct parameters' do - expect(tool.parameters).to have_key(:reason) - expect(tool.parameters[:reason].name).to eq(:reason) - expect(tool.parameters[:reason].type).to eq('string') - expect(tool.parameters[:reason].description).to eq('The reason why handoff is needed (optional)') - expect(tool.parameters[:reason].required).to be false + describe '#params_schema' do + it 'constrains the reason category to the supported values and keeps the reason optional' do + schema = tool.params_schema + + expect(schema['properties']['reason']['type']).to eq('string') + expect(schema['properties']['reason_category']['enum']).to eq(described_class::REASON_CATEGORIES) + expect(schema['required']).to eq(['reason_category']) + end + + it 'uses only reason categories supported by conversation outcomes' do + expect(ConversationOutcome::HANDOFF_REASON_CATEGORIES).to include(*described_class::REASON_CATEGORIES) + end + + it 'survives Agents::ToolWrapper reading the class params at wrap time' do + described_class.params + + expect(described_class.new(assistant).params_schema['properties'].keys).to contain_exactly('reason', 'reason_category') end end @@ -96,9 +106,16 @@ RSpec.describe Captain::Tools::HandoffTool, type: :model do it 'emits a captain handoff event with the tool source after the locked handoff completes' do expect(Captain::ConversationEvents).to receive(:handed_off) - .with(conversation: conversation, assistant: assistant, source: 'tool', at: kind_of(Time)) + .with(conversation: conversation, assistant: assistant, source: 'tool', reason_category: 'customer_request', at: kind_of(Time)) - tool.perform(tool_context, reason: 'Customer needs specialized support') + tool.perform(tool_context, reason: 'Customer needs specialized support', reason_category: 'customer_request') + end + + it 'emits an unclassified handoff when the model supplies an unknown reason category' do + expect(Captain::ConversationEvents).to receive(:handed_off) + .with(conversation: conversation, assistant: assistant, source: 'tool', reason_category: nil, at: kind_of(Time)) + + tool.perform(tool_context, reason: 'Customer needs specialized support', reason_category: 'hallucinated_category') end it 'does not emit a captain handoff event when the handoff is skipped as stale' do @@ -170,11 +187,33 @@ RSpec.describe Captain::Tools::HandoffTool, type: :model do tool.perform(tool_context, reason: 'Test reason') end - it 'emits a captain handoff event with the tool source' do + it 'emits a captain handoff event with the tool source and reason category' do expect(Captain::ConversationEvents).to receive(:handed_off) - .with(conversation: conversation, assistant: assistant, source: 'tool', at: kind_of(Time)) + .with(conversation: conversation, assistant: assistant, source: 'tool', reason_category: 'unsupported_request', at: kind_of(Time)) - tool.perform(tool_context, reason: 'Test reason') + tool.perform(tool_context, reason: 'Test reason', reason_category: 'unsupported_request') + end + + it 'records the handoff on an existing V2 outcome' do + account.enable_features!('captain_integration_v2') + create( + :conversation_outcome, + account: account, + assistant: assistant, + conversation: conversation, + inbox: inbox + ) + + tool.perform( + tool_context, + reason: 'Customer needs specialized support', + reason_category: 'unsupported_request' + ) + + expect(ConversationOutcome.last).to have_attributes( + handoff_reason_category: 'unsupported_request', + handoff_at: be_present + ) end it 'creates a conversation_bot_handoff reporting event' do diff --git a/spec/enterprise/listeners/captain_listener_spec.rb b/spec/enterprise/listeners/captain_listener_spec.rb index 4e25c40c4..060fce2ec 100644 --- a/spec/enterprise/listeners/captain_listener_spec.rb +++ b/spec/enterprise/listeners/captain_listener_spec.rb @@ -11,7 +11,7 @@ describe CaptainListener do let(:conversation) { create(:conversation, account: account, inbox: inbox, assignee: user) } let(:event_name) { :conversation_resolved } - let(:event) { Events::Base.new(event_name, Time.zone.now, conversation: conversation) } + let(:event) { Events::Base.new(event_name, Time.zone.now.change(usec: 0), conversation: conversation) } before do create(:captain_inbox, captain_assistant: assistant, inbox: inbox) @@ -49,5 +49,74 @@ describe CaptainListener do listener.conversation_resolved(event) end end + + it 'records the resolution on an existing V2 outcome' do + assistant.update!(config: {}) + outcome = create( + :conversation_outcome, + account: account, + assistant: assistant, + conversation: conversation, + inbox: inbox, + started_at: 10.minutes.ago + ) + captain_reply = create( + :message, + account: account, + inbox: inbox, + conversation: conversation, + sender: assistant, + message_type: :outgoing, + created_at: 5.minutes.ago.change(usec: 0) + ) + + listener.conversation_resolved(event) + + expect(outcome.reload).to have_attributes( + captain_reply_count: 1, + first_captain_reply_at: captain_reply.created_at, + resolved_at: event.timestamp + ) + end + end + + describe '#message_updated' do + let(:conversation) { create(:conversation, account: account, inbox: inbox) } + let!(:outcome) do + create( + :conversation_outcome, + account: account, + assistant: assistant, + conversation: conversation, + inbox: inbox + ) + end + + it 'records a submitted CSAT response' do + message = create( + :message, + account: account, + inbox: inbox, + conversation: conversation, + content_type: :input_csat, + message_type: :outgoing + ) + response = create( + :csat_survey_response, + account: account, + conversation: conversation, + contact: conversation.contact, + message: message, + rating: 5 + ) + event = Events::Base.new(:message_updated, Time.current, message: message) + + listener.message_updated(event) + + expect(outcome.reload).to have_attributes( + csat_rating: 5, + csat_received_at: response.created_at + ) + end end end diff --git a/spec/enterprise/services/captain/conversation_events_spec.rb b/spec/enterprise/services/captain/conversation_events_spec.rb index b6f22d6cb..5ab854578 100644 --- a/spec/enterprise/services/captain/conversation_events_spec.rb +++ b/spec/enterprise/services/captain/conversation_events_spec.rb @@ -6,18 +6,6 @@ RSpec.describe Captain::ConversationEvents do let(:assistant) { create(:captain_assistant, account: account) } let(:timestamp) { Time.zone.now } - describe '.engaged' do - it 'dispatches the engagement event with normalized context' do - expect(Rails.configuration.dispatcher).to receive(:dispatch).with( - Events::Types::CAPTAIN_CONVERSATION_ENGAGED, - timestamp, - { conversation: conversation, assistant: assistant } - ) - - described_class.engaged(conversation: conversation, assistant: assistant, at: timestamp) - end - end - describe '.handed_off' do it 'dispatches the handoff event with source and reason category' do expect(Rails.configuration.dispatcher).to receive(:dispatch).with( @@ -69,7 +57,9 @@ RSpec.describe Captain::ConversationEvents do .and_return(instance_double(ChatwootExceptionTracker, capture_exception: true)) expect do - described_class.engaged(conversation: conversation, assistant: assistant, at: timestamp) + described_class.handed_off( + conversation: conversation, assistant: assistant, source: 'tool', reason_category: nil, at: timestamp + ) end.not_to raise_error end end diff --git a/spec/enterprise/services/captain/conversation_outcome_tracker_spec.rb b/spec/enterprise/services/captain/conversation_outcome_tracker_spec.rb new file mode 100644 index 000000000..9112283ce --- /dev/null +++ b/spec/enterprise/services/captain/conversation_outcome_tracker_spec.rb @@ -0,0 +1,195 @@ +require 'rails_helper' + +RSpec.describe Captain::ConversationOutcomeTracker do + let(:account) { create(:account) } + let(:assistant) { create(:captain_assistant, account: account) } + let(:inbox) { create(:inbox, account: account) } + let(:conversation) { create(:conversation, account: account, inbox: inbox, status: :pending) } + let(:tracker) { described_class.new(conversation: conversation, assistant: assistant) } + + def episodes + ConversationOutcome.where(conversation_id: conversation.id).order(:started_at) + end + + before do + account.enable_features!('captain_integration_v2') + create(:captain_inbox, captain_assistant: assistant, inbox: inbox) + end + + describe '#record_eligibility' do + it 'creates the initial episode anchored to the demand time' do + eligible_at = 2.minutes.ago.change(usec: 0) + + episode = tracker.record_eligibility(at: eligible_at) + + expect(episode.reload).to have_attributes( + account: account, + assistant: assistant, + conversation: conversation, + inbox: inbox, + episode_trigger: 'initial', + started_at: eligible_at, + ended_at: nil + ) + end + + it 'is idempotent' do + tracker.record_eligibility(at: Time.current) + + expect do + tracker.record_eligibility(at: Time.current) + end.not_to change(ConversationOutcome, :count) + end + end + + describe '#record_reopen' do + let(:initial_at) { 30.minutes.ago.change(usec: 0) } + let!(:initial) { tracker.record_eligibility(at: initial_at) } + + it 'closes the open episode and opens a reopen episode' do + boundary_at = 5.minutes.ago.change(usec: 0) + + tracker.record_reopen(at: boundary_at) + + expect(initial.reload.ended_at).to eq(boundary_at) + expect(episodes.last).to have_attributes( + episode_trigger: 'reopen', + assistant: assistant, + started_at: boundary_at, + ended_at: nil + ) + end + + it 'creates exactly one episode per boundary regardless of history' do + tracker.record_reopen(at: 20.minutes.ago) + tracker.record_reopen(at: 10.minutes.ago) + + expect do + tracker.record_reopen(at: 1.minute.ago) + end.to change(ConversationOutcome, :count).by(1) + end + + it 'ignores boundaries for conversations without Captain history' do + ConversationOutcome.delete_all + + expect do + tracker.record_reopen(at: Time.current) + end.not_to change(ConversationOutcome, :count) + end + + it 'captures boundary failures without raising' do + error = ActiveRecord::StatementInvalid.new('database unavailable') + exception_tracker = instance_double(ChatwootExceptionTracker) + allow(ConversationOutcome).to receive(:create!).and_raise(error) + expect(ChatwootExceptionTracker).to receive(:new).with(error, account: account).and_return(exception_tracker) + expect(exception_tracker).to receive(:capture_exception) + + expect do + tracker.record_reopen(at: Time.current) + end.not_to raise_error + end + end + + describe '#record_handoff' do + let!(:initial) { tracker.record_eligibility(at: 30.minutes.ago) } + + it 'snapshots message facts and records the handoff reason' do + first_reply = create( + :message, + account: account, inbox: inbox, conversation: conversation, + sender: assistant, message_type: :outgoing, created_at: 20.minutes.ago.change(usec: 0) + ) + last_reply = create( + :message, + account: account, inbox: inbox, conversation: conversation, + sender: assistant, message_type: :outgoing, created_at: 10.minutes.ago.change(usec: 0) + ) + handoff_at = 5.minutes.ago.change(usec: 0) + tracker.record_handoff(at: handoff_at, reason_category: 'missing_knowledge') + + expect(initial.reload).to have_attributes( + captain_reply_count: 2, + first_captain_reply_at: first_reply.created_at, + last_captain_reply_at: last_reply.created_at, + handoff_at: handoff_at, + handoff_reason_category: 'missing_knowledge' + ) + end + end + + describe '#record_resolution' do + let!(:initial) { tracker.record_eligibility(at: 30.minutes.ago) } + + it 'snapshots the episode active at the resolution event time' do + captain_reply = create( + :message, + account: account, inbox: inbox, conversation: conversation, + sender: assistant, message_type: :outgoing, created_at: 20.minutes.ago.change(usec: 0) + ) + resolved_at = 15.minutes.ago.change(usec: 0) + tracker.record_reopen(at: 10.minutes.ago.change(usec: 0)) + + tracker.record_resolution(at: resolved_at) + + expect(initial.reload).to have_attributes( + captain_reply_count: 1, + first_captain_reply_at: captain_reply.created_at, + resolved_at: resolved_at + ) + expect(episodes.last.resolved_at).to be_nil + end + + it 'snapshots Captain and human replies through resolution' do + captain_reply = create( + :message, + account: account, inbox: inbox, conversation: conversation, + sender: assistant, message_type: :outgoing, created_at: 20.minutes.ago.change(usec: 0) + ) + agent = create(:user, account: account) + create( + :message, + account: account, inbox: inbox, conversation: conversation, + sender: agent, message_type: :outgoing, created_at: 15.minutes.ago, + content_attributes: { automation_rule_id: 1 } + ) + human_reply = create( + :message, + account: account, inbox: inbox, conversation: conversation, + sender: agent, message_type: :outgoing, created_at: 10.minutes.ago.change(usec: 0) + ) + resolved_at = 5.minutes.ago.change(usec: 0) + + tracker.record_resolution(at: resolved_at) + + expect(initial.reload).to have_attributes( + captain_reply_count: 1, + first_captain_reply_at: captain_reply.created_at, + last_captain_reply_at: captain_reply.created_at, + first_human_reply_at: human_reply.created_at, + resolved_at: resolved_at + ) + end + end + + describe '#record_csat' do + let!(:initial) { tracker.record_eligibility(at: 30.minutes.ago) } + + it 'attributes the response to the episode that issued its survey' do + survey_message = create( + :message, + account: account, inbox: inbox, conversation: conversation, + content_type: :input_csat, created_at: 20.minutes.ago + ) + tracker.record_reopen(at: 10.minutes.ago) + response = create( + :csat_survey_response, + account: account, conversation: conversation, message: survey_message, rating: 4 + ) + + tracker.record_csat(response: response) + + expect(initial.reload).to have_attributes(csat_rating: 4, csat_received_at: response.created_at) + expect(episodes.last.csat_rating).to be_nil + end + end +end diff --git a/spec/enterprise/services/enterprise/message_templates/hook_execution_service_spec.rb b/spec/enterprise/services/enterprise/message_templates/hook_execution_service_spec.rb index f4974c1b5..118cbdeda 100644 --- a/spec/enterprise/services/enterprise/message_templates/hook_execution_service_spec.rb +++ b/spec/enterprise/services/enterprise/message_templates/hook_execution_service_spec.rb @@ -137,19 +137,23 @@ RSpec.describe MessageTemplates::HookExecutionService do create(:message, conversation: conversation, message_type: :incoming, account: account) end - it 'emits the engagement event when captain V2 is enabled' do + it 'records a conversation outcome when captain V2 is enabled' do account.enable_features!('captain_integration_v2') - expect(Captain::ConversationEvents).to receive(:engaged) - .with(conversation: conversation, assistant: assistant, at: kind_of(Time)) + expect do + create(:message, conversation: conversation, message_type: :incoming, account: account) + end.to change(ConversationOutcome, :count).by(1) - create(:message, conversation: conversation, message_type: :incoming, account: account) + expect(ConversationOutcome.last).to have_attributes( + assistant: assistant, + conversation: conversation + ) end - it 'does not emit the engagement event when captain V2 is disabled' do - expect(Captain::ConversationEvents).not_to receive(:engaged) - - create(:message, conversation: conversation, message_type: :incoming, account: account) + it 'does not record a conversation outcome when captain V2 is disabled' do + expect do + create(:message, conversation: conversation, message_type: :incoming, account: account) + end.not_to change(ConversationOutcome, :count) end end @@ -173,19 +177,30 @@ RSpec.describe MessageTemplates::HookExecutionService do expect(conversation.reload.status).to eq('open') end - it 'emits a usage limit handoff event when captain V2 is enabled' do - account.enable_features!('captain_integration_v2') - + it 'emits a usage limit handoff event' do expect(Captain::ConversationEvents).to receive(:handed_off) .with(conversation: conversation, assistant: assistant, source: 'usage_limit', reason_category: :usage_limit, at: kind_of(Time)) create(:message, conversation: conversation, message_type: :incoming, account: account) end - it 'does not emit a handoff event when captain V2 is disabled' do - expect(Captain::ConversationEvents).not_to receive(:handed_off) + it 'records the handoff on the outcome when captain V2 is enabled' do + account.enable_features!('captain_integration_v2') - create(:message, conversation: conversation, message_type: :incoming, account: account) + expect do + create(:message, conversation: conversation, message_type: :incoming, account: account) + end.to change(ConversationOutcome, :count).by(1) + + expect(ConversationOutcome.last).to have_attributes( + handoff_reason_category: 'usage_limit', + handoff_at: be_present + ) + end + + it 'does not record an outcome when captain V2 is disabled' do + expect do + create(:message, conversation: conversation, message_type: :incoming, account: account) + end.not_to change(ConversationOutcome, :count) end end end @@ -222,6 +237,14 @@ RSpec.describe MessageTemplates::HookExecutionService do create(:message, conversation: conversation, message_type: :incoming, account: account) end + + it 'still records the conversation as eligible demand when captain V2 is enabled' do + account.enable_features!('captain_integration_v2') + + expect do + create(:message, conversation: conversation, message_type: :incoming, account: account) + end.to change(ConversationOutcome, :count).by(1) + end end context 'when the contact is inside the assistant audience' do