From 7981e2cc75c8e51f19765ef5fb2879a2c910c106 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 31 Jul 2026 13:55:57 +0530 Subject: [PATCH] refactor: introduce normalized Captain lifecycle events [CW-7792] (#15213) This introduces a small event layer for the Captain V2 conversation lifecycle. A new `Captain::ConversationEvents` facade dispatches five normalized events (`captain.conversation.engaged`, `captain.conversation.handed_off`, `captain.conversation.resolved`, `captain.response.completed`, `captain.response.failed`) from the points where Captain engages a conversation, replies, fails, hands off, or auto-resolves. Each event carries the conversation, assistant, timestamp, and a `source`/`reason_category` where relevant. ## Why this, why now The Captain V2 flow is about to gain several observers at once: conversation outcome tracking, agent session capture, and analytics all need to know when Captain engages, replies, fails, hands off, or resolves. Wiring each of them directly into `ResponseBuilderJob`, `HookExecutionService`, and the tools would tangle secondary bookkeeping into the paths that deliver customer-facing behavior, and every future consumer would deepen that. Landing the event layer first as its own PR means the flow announces these moments once and stays otherwise untouched: customer-visible behavior (messages, status changes, handoffs, usage enforcement) remains synchronous, while secondary effects subscribe through listeners. The upcoming conversation outcomes PR then reduces to a listener plus a model instead of another round of edits to the core flow, which is why this ships now, before that work merges. The existing inference reporting behavior is folded into this layer: the `conversation.captain_inference_*` events and their dispatch helpers on `Enterprise::Conversation` are removed, and a dedicated `Captain::ReportingEventListener` (registered on the enterprise async dispatcher) maps `source: 'inference'` events to the same stored reporting event names, so recorded analytics and the assistant stats builder are unaffected. ## What changed - New `Captain::ConversationEvents` facade and event type constants - Event emission from `HookExecutionService` (engagement, usage-limit handoff), `ResponseBuilderJob` (response completed/failed, generation-failure handoff), `HandoffTool` (tool handoff), and `InboxPendingConversationsResolutionJob` (inference resolved/handoff) - A dedicated `Captain::ReportingEventListener` preserves inference reporting events through the new event names, removing captain logic from the OSS listener --- app/listeners/reporting_event_listener.rb | 24 ----- .../enterprise/async_dispatcher.rb | 3 +- .../conversation/response_builder_job.rb | 39 ++++++-- .../conversation/v2_lifecycle_events.rb | 30 +++++++ ...ox_pending_conversations_resolution_job.rb | 21 ++++- .../captain/reporting_event_listener.rb | 27 ++++++ .../app/models/enterprise/conversation.rb | 12 --- .../captain/assistant/agent_runner_service.rb | 8 +- .../services/captain/conversation_events.rb | 72 +++++++++++++++ .../hook_execution_service.rb | 24 +++++ enterprise/lib/captain/tools/handoff_tool.rb | 7 ++ .../tools/resolve_conversation_tool.rb | 2 + lib/events/types.rb | 7 +- .../reports/assistant_conversation_creator.rb | 9 +- .../conversation/response_builder_job_spec.rb | 65 ++++++++++++++ ...nding_conversations_resolution_job_spec.rb | 31 +++++++ .../lib/captain/tools/handoff_tool_spec.rb | 22 +++++ .../tools/resolve_conversation_tool_spec.rb | 13 +++ .../captain/reporting_event_listener_spec.rb | 53 +++++++++++ .../assistant/agent_runner_service_spec.rb | 6 ++ .../captain/conversation_events_spec.rb | 88 +++++++++++++++++++ .../hook_execution_service_spec.rb | 30 +++++++ .../reporting_event_listener_spec.rb | 32 ------- 23 files changed, 540 insertions(+), 85 deletions(-) create mode 100644 enterprise/app/jobs/captain/conversation/v2_lifecycle_events.rb create mode 100644 enterprise/app/listeners/captain/reporting_event_listener.rb create mode 100644 enterprise/app/services/captain/conversation_events.rb create mode 100644 spec/enterprise/listeners/captain/reporting_event_listener_spec.rb create mode 100644 spec/enterprise/services/captain/conversation_events_spec.rb diff --git a/app/listeners/reporting_event_listener.rb b/app/listeners/reporting_event_listener.rb index 40aa0b0ac..d8137e727 100644 --- a/app/listeners/reporting_event_listener.rb +++ b/app/listeners/reporting_event_listener.rb @@ -98,14 +98,6 @@ class ReportingEventListener < BaseListener safe_rollup(reporting_event) end - def conversation_captain_inference_resolved(event) - create_captain_inference_event(event, 'conversation_captain_inference_resolved') - end - - def conversation_captain_inference_handoff(event) - create_captain_inference_event(event, 'conversation_captain_inference_handoff') - end - def conversation_opened(event) conversation = extract_conversation_and_account(event)[0] event_end_time = event.timestamp @@ -148,22 +140,6 @@ class ReportingEventListener < BaseListener reporting_event.save! end - def create_captain_inference_event(event, event_name) - conversation = extract_conversation_and_account(event)[0] - time_to_event = event.timestamp.to_i - conversation.created_at.to_i - - ReportingEvent.create!( - name: event_name, - value: time_to_event, - account_id: conversation.account_id, - inbox_id: conversation.inbox_id, - user_id: conversation.assignee_id, - conversation_id: conversation.id, - event_start_time: conversation.created_at, - event_end_time: event.timestamp - ) - end - def create_bot_resolved_event(conversation, reporting_event) return unless conversation.inbox.active_bot? # We don't want to create a bot_resolved event if there is user interaction on the conversation diff --git a/enterprise/app/dispatchers/enterprise/async_dispatcher.rb b/enterprise/app/dispatchers/enterprise/async_dispatcher.rb index 0225e2d04..7fe65143e 100644 --- a/enterprise/app/dispatchers/enterprise/async_dispatcher.rb +++ b/enterprise/app/dispatchers/enterprise/async_dispatcher.rb @@ -1,7 +1,8 @@ module Enterprise::AsyncDispatcher def listeners super + [ - CaptainListener.instance + CaptainListener.instance, + Captain::ReportingEventListener.instance ] end end diff --git a/enterprise/app/jobs/captain/conversation/response_builder_job.rb b/enterprise/app/jobs/captain/conversation/response_builder_job.rb index b86f859f3..f37086816 100644 --- a/enterprise/app/jobs/captain/conversation/response_builder_job.rb +++ b/enterprise/app/jobs/captain/conversation/response_builder_job.rb @@ -1,6 +1,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob include Captain::Conversation::V1ActionClassifier include Captain::Conversation::V1FalsePromiseHandler + include Captain::Conversation::V2LifecycleEvents include Captain::Conversation::MessageBuilder MAX_MESSAGE_LENGTH = 10_000 @@ -64,20 +65,30 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob end def process_response + # The V2 runner rescues its own generation errors and signals them via an error + # response instead of raising, so the failure event must be emitted here — the + # top-level handle_error path only sees exceptions raised outside the runner. + record_v2_response_failure(@response['error_reason']) if v2_generation_errored? + if v2_handoff_tool_fired? process_v2_handoff_response elsif v1_handoff_requested? - # V1 only signals via the response string — no state has been touched yet. If - # the conversation isn't pending anymore, a human took over mid-run; bail out - # rather than posting a stale handoff message on top of their reply. - return unless conversation_pending? - - process_v1_handoff + process_v1_handoff_request elsif conversation_pending? process_standard_response end end + def process_v1_handoff_request + # V1 only signals via the response string — no state has been touched yet. If + # the conversation isn't pending anymore, a human took over mid-run; bail out + # rather than posting a stale handoff message on top of their reply. + return unless conversation_pending? + + process_v1_handoff + record_v2_failure_handoff if v2_generation_errored? + end + def process_standard_response message = nil ActiveRecord::Base.transaction do @@ -90,6 +101,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob return unless message capture_assistant_session(result_message: message, credits_consumed: 1.0) + record_v2_response_completed(message) if captain_v2_enabled? end def process_v2_handoff_response @@ -98,6 +110,10 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob if captain_v2_enabled? return unless v2_handoff_tool_completed? || conversation_pending? + # Known gap, accepted for now: the fallback V1 handoff (tool fired but never + # completed) emits no captain.conversation.handed_off event — the tool emits + # only after a successful bot_handoff!. If outcome data ever needs it, emit + # here with a distinct source such as 'tool_fallback'. v2_handoff_tool_completed? ? process_v2_handoff : process_v1_handoff else conversation_pending? ? process_v1_handoff : process_v2_handoff @@ -173,10 +189,19 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob @response ||= {} @response['action_source'] ||= 'error' @response['action_reason'] ||= error_action_reason(error) - process_v1_handoff if conversation_pending? && (!captain_v2_enabled? || !newer_customer_message_arrived?) + record_v2_response_failure(error_action_reason(error)) if captain_v2_enabled? + process_error_handoff true end + def process_error_handoff + return unless conversation_pending? + return if captain_v2_enabled? && newer_customer_message_arrived? + + process_v1_handoff + record_v2_failure_handoff if captain_v2_enabled? + end + def log_error(error) ChatwootExceptionTracker.new(error, account: account).capture_exception end diff --git a/enterprise/app/jobs/captain/conversation/v2_lifecycle_events.rb b/enterprise/app/jobs/captain/conversation/v2_lifecycle_events.rb new file mode 100644 index 000000000..37501e9ff --- /dev/null +++ b/enterprise/app/jobs/captain/conversation/v2_lifecycle_events.rb @@ -0,0 +1,30 @@ +module Captain::Conversation::V2LifecycleEvents + private + + def v2_generation_errored? + captain_v2_enabled? && @response['error'].present? + end + + def record_v2_response_completed(message) + Captain::ConversationEvents.response_completed( + conversation: @conversation, + assistant: @assistant, + message: message, + at: Time.current + ) + end + + def record_v2_response_failure(reason) + Captain::ConversationEvents.response_failed(conversation: @conversation, assistant: @assistant, reason: reason, at: Time.current) + end + + def record_v2_failure_handoff + Captain::ConversationEvents.handed_off( + conversation: @conversation, + assistant: @assistant, + source: Captain::ConversationEvents::Sources::GENERATION_FAILURE, + reason_category: :tool_failure, + at: Time.current + ) + end +end diff --git a/enterprise/app/jobs/captain/inbox_pending_conversations_resolution_job.rb b/enterprise/app/jobs/captain/inbox_pending_conversations_resolution_job.rb index 0be179f04..958c809bf 100644 --- a/enterprise/app/jobs/captain/inbox_pending_conversations_resolution_job.rb +++ b/enterprise/app/jobs/captain/inbox_pending_conversations_resolution_job.rb @@ -28,6 +28,12 @@ class Captain::InboxPendingConversationsResolutionJob < ApplicationJob resolvable_pending_conversations(inbox).each do |conversation| create_resolution_message(conversation, inbox) conversation.resolved! + Captain::ConversationEvents.resolved( + conversation: conversation, + assistant: inbox.captain_assistant, + source: Captain::ConversationEvents::Sources::TIME_BASED, + at: Time.current + ) end end @@ -77,7 +83,12 @@ class Captain::InboxPendingConversationsResolutionJob < ApplicationJob reason: CAPTAIN_INFERENCE_RESOLVE_ACTIVITY_REASON, reason_type: :inference ) { conversation.resolved! } - conversation.dispatch_captain_inference_resolved_event + Captain::ConversationEvents.resolved( + conversation: conversation, + assistant: inbox.captain_assistant, + source: Captain::ConversationEvents::Sources::INFERENCE, + at: Time.current + ) end def handoff_conversation(conversation, inbox, reason) @@ -87,7 +98,13 @@ class Captain::InboxPendingConversationsResolutionJob < ApplicationJob reason: CAPTAIN_INFERENCE_HANDOFF_ACTIVITY_REASON, reason_type: :inference ) { conversation.bot_handoff! } - conversation.dispatch_captain_inference_handoff_event + Captain::ConversationEvents.handed_off( + conversation: conversation, + assistant: inbox.captain_assistant, + source: Captain::ConversationEvents::Sources::INFERENCE, + reason_category: :pending_clarification, + at: Time.current + ) send_out_of_office_message_if_applicable(conversation.reload) end diff --git a/enterprise/app/listeners/captain/reporting_event_listener.rb b/enterprise/app/listeners/captain/reporting_event_listener.rb new file mode 100644 index 000000000..6033dca45 --- /dev/null +++ b/enterprise/app/listeners/captain/reporting_event_listener.rb @@ -0,0 +1,27 @@ +class Captain::ReportingEventListener < BaseListener + def captain_conversation_handed_off(event) + create_captain_inference_event(event, 'conversation_captain_inference_handoff') if event.data[:source] == 'inference' + end + + def captain_conversation_resolved(event) + create_captain_inference_event(event, 'conversation_captain_inference_resolved') if event.data[:source] == 'inference' + end + + private + + def create_captain_inference_event(event, event_name) + conversation = extract_conversation_and_account(event)[0] + time_to_event = event.timestamp.to_i - conversation.created_at.to_i + + ReportingEvent.create!( + name: event_name, + value: time_to_event, + account_id: conversation.account_id, + inbox_id: conversation.inbox_id, + user_id: conversation.assignee_id, + conversation_id: conversation.id, + event_start_time: conversation.created_at, + event_end_time: event.timestamp + ) + end +end diff --git a/enterprise/app/models/enterprise/conversation.rb b/enterprise/app/models/enterprise/conversation.rb index 0765e7279..f57aa3f05 100644 --- a/enterprise/app/models/enterprise/conversation.rb +++ b/enterprise/app/models/enterprise/conversation.rb @@ -1,14 +1,6 @@ module Enterprise::Conversation attr_accessor :captain_activity_reason, :captain_activity_reason_type - def dispatch_captain_inference_resolved_event - dispatch_captain_inference_event(Events::Types::CONVERSATION_CAPTAIN_INFERENCE_RESOLVED) - end - - def dispatch_captain_inference_handoff_event - dispatch_captain_inference_event(Events::Types::CONVERSATION_CAPTAIN_INFERENCE_HANDOFF) - end - def list_of_keys super + %w[sla_policy_id] end @@ -50,10 +42,6 @@ module Enterprise::Conversation current_applied_sla.update!(completed_at: resolved? ? Time.current : nil) end - def dispatch_captain_inference_event(event_name) - dispatcher_dispatch(event_name) - end - def call_attributes_changed? return false if previous_changes['additional_attributes'].blank? diff --git a/enterprise/app/services/captain/assistant/agent_runner_service.rb b/enterprise/app/services/captain/assistant/agent_runner_service.rb index 0d3d52e86..b2c7efa45 100644 --- a/enterprise/app/services/captain/assistant/agent_runner_service.rb +++ b/enterprise/app/services/captain/assistant/agent_runner_service.rb @@ -34,7 +34,7 @@ class Captain::Assistant::AgentRunnerService Rails.logger.error "[Captain V2] AgentRunnerService error: #{e.message}" Rails.logger.error e.backtrace.join("\n") - error_response(e.message) + error_response(e) end def response_discarded? = @response_discarded == true @@ -132,10 +132,12 @@ class Captain::Assistant::AgentRunnerService @message_length_limit ||= Captain::MessageLengthLimit.for(@conversation) end - def error_response(error_message) + def error_response(error) { 'response' => 'conversation_handoff', - 'reasoning' => "Error occurred: #{error_message}", + 'reasoning' => "Error occurred: #{error.message}", + 'error' => true, + 'error_reason' => error.class.name.underscore.tr('/', '_'), 'handoff_tool_called' => @handoff_tool_called } end diff --git a/enterprise/app/services/captain/conversation_events.rb b/enterprise/app/services/captain/conversation_events.rb new file mode 100644 index 000000000..964b2d365 --- /dev/null +++ b/enterprise/app/services/captain/conversation_events.rb @@ -0,0 +1,72 @@ +class Captain::ConversationEvents + module Sources + TOOL = 'tool'.freeze + GENERATION_FAILURE = 'generation_failure'.freeze + TIME_BASED = 'time_based'.freeze + INFERENCE = 'inference'.freeze + USAGE_LIMIT = 'usage_limit'.freeze + end + + class << self + def engaged(conversation:, assistant:, at:) + dispatch( + Events::Types::CAPTAIN_CONVERSATION_ENGAGED, + at: at, + conversation: conversation, + assistant: assistant + ) + end + + def handed_off(conversation:, assistant:, source:, at:, reason_category: nil) + dispatch( + Events::Types::CAPTAIN_CONVERSATION_HANDED_OFF, + at: at, + conversation: conversation, + assistant: assistant, + source: source, + reason_category: reason_category + ) + end + + def resolved(conversation:, assistant:, source:, at:) + dispatch( + Events::Types::CAPTAIN_CONVERSATION_RESOLVED, + at: at, + conversation: conversation, + assistant: assistant, + source: source + ) + end + + def response_completed(conversation:, assistant:, message:, at:) + dispatch( + Events::Types::CAPTAIN_RESPONSE_COMPLETED, + at: at, + conversation: conversation, + assistant: assistant, + message: message + ) + end + + def response_failed(conversation:, assistant:, reason:, at:) + dispatch( + Events::Types::CAPTAIN_RESPONSE_FAILED, + at: at, + conversation: conversation, + assistant: assistant, + reason: reason + ) + end + + private + + # Lifecycle events are secondary effects: a dispatch failure must never alter + # the customer-facing Captain flow (blocking a response from being scheduled, + # or turning an already-delivered reply into a spurious handoff). + def dispatch(event_name, at:, **data) + Rails.configuration.dispatcher.dispatch(event_name, at, data) + rescue StandardError => e + ChatwootExceptionTracker.new(e, account: data[:conversation]&.account).capture_exception + end + end +end diff --git a/enterprise/app/services/enterprise/message_templates/hook_execution_service.rb b/enterprise/app/services/enterprise/message_templates/hook_execution_service.rb index d32968384..b2491607e 100644 --- a/enterprise/app/services/enterprise/message_templates/hook_execution_service.rb +++ b/enterprise/app/services/enterprise/message_templates/hook_execution_service.rb @@ -6,6 +6,7 @@ module Enterprise::MessageTemplates::HookExecutionService return unless should_process_captain_response? return perform_handoff unless inbox.captain_active? + track_captain_engagement schedule_captain_response end @@ -29,6 +30,20 @@ module Enterprise::MessageTemplates::HookExecutionService private + def track_captain_engagement + return unless captain_v2_enabled? + + Captain::ConversationEvents.engaged( + conversation: conversation, + assistant: inbox.captain_assistant, + at: message.created_at + ) + end + + def captain_v2_enabled? + conversation.account.feature_enabled?('captain_integration_v2') + end + def schedule_captain_response job_args = [conversation, conversation.inbox.captain_assistant] captain_v2_enabled = conversation.account.feature_enabled?('captain_integration_v2') @@ -81,6 +96,15 @@ module Enterprise::MessageTemplates::HookExecutionService content: 'Transferring to another agent for further assistance.' ) conversation.bot_handoff! + if captain_v2_enabled? + Captain::ConversationEvents.handed_off( + conversation: conversation, + assistant: inbox.captain_assistant, + source: Captain::ConversationEvents::Sources::USAGE_LIMIT, + reason_category: :usage_limit, + at: Time.current + ) + end send_out_of_office_message_after_handoff end diff --git a/enterprise/lib/captain/tools/handoff_tool.rb b/enterprise/lib/captain/tools/handoff_tool.rb index 73a76c2d0..e1d8a11c8 100644 --- a/enterprise/lib/captain/tools/handoff_tool.rb +++ b/enterprise/lib/captain/tools/handoff_tool.rb @@ -54,6 +54,7 @@ class Captain::Tools::HandoffTool < Captain::Tools::BasePublicTool tool_context.state[:captain_v2_handoff_tool_completed] = true # Queue the event after the state change commits so notification jobs always see the open conversation. conversation.dispatch_bot_handoff_event + emit_tool_handoff_event(conversation) # Send out of office message if applicable (since template messages were suppressed while Captain was handling) send_out_of_office_message_if_applicable(conversation) @@ -67,10 +68,16 @@ class Captain::Tools::HandoffTool < Captain::Tools::BasePublicTool ) record_handoff_note(tool_context, note) if reason.present? conversation.bot_handoff! + emit_tool_handoff_event(conversation) send_out_of_office_message_if_applicable(conversation) :completed end + def emit_tool_handoff_event(conversation) + Captain::ConversationEvents.handed_off(conversation: conversation, assistant: @assistant, + source: Captain::ConversationEvents::Sources::TOOL, at: Time.current) + end + def record_handoff_note(tool_context, note) metadata = tool_context.state[:cw_metadata] ||= {} metadata[:handoff_note_id] = note.id diff --git a/enterprise/lib/captain/tools/resolve_conversation_tool.rb b/enterprise/lib/captain/tools/resolve_conversation_tool.rb index eeffc3f06..ba4d8323a 100644 --- a/enterprise/lib/captain/tools/resolve_conversation_tool.rb +++ b/enterprise/lib/captain/tools/resolve_conversation_tool.rb @@ -11,6 +11,8 @@ class Captain::Tools::ResolveConversationTool < Captain::Tools::BasePublicTool log_tool_usage('resolve_conversation', { conversation_id: conversation.id, reason: reason }) conversation.with_captain_activity_context(reason: reason, reason_type: :tool) { conversation.resolved! } + Captain::ConversationEvents.resolved(conversation: conversation, assistant: @assistant, + source: Captain::ConversationEvents::Sources::TOOL, at: Time.current) "Conversation ##{conversation.display_id} resolved#{" (Reason: #{reason})" if reason}" end diff --git a/lib/events/types.rb b/lib/events/types.rb index 171649a7d..653739826 100644 --- a/lib/events/types.rb +++ b/lib/events/types.rb @@ -22,8 +22,11 @@ module Events::Types # FIXME: deprecate the opened and resolved events in future in favor of status changed event. CONVERSATION_OPENED = 'conversation.opened' CONVERSATION_RESOLVED = 'conversation.resolved' - CONVERSATION_CAPTAIN_INFERENCE_RESOLVED = 'conversation.captain_inference_resolved' - CONVERSATION_CAPTAIN_INFERENCE_HANDOFF = 'conversation.captain_inference_handoff' + CAPTAIN_CONVERSATION_ENGAGED = 'captain.conversation.engaged' + CAPTAIN_CONVERSATION_HANDED_OFF = 'captain.conversation.handed_off' + CAPTAIN_CONVERSATION_RESOLVED = 'captain.conversation.resolved' + CAPTAIN_RESPONSE_COMPLETED = 'captain.response.completed' + CAPTAIN_RESPONSE_FAILED = 'captain.response.failed' CONVERSATION_STATUS_CHANGED = 'conversation.status_changed' CONVERSATION_CONTACT_CHANGED = 'conversation.contact_changed' diff --git a/lib/seeders/reports/assistant_conversation_creator.rb b/lib/seeders/reports/assistant_conversation_creator.rb index 9462d899e..d21e05976 100644 --- a/lib/seeders/reports/assistant_conversation_creator.rb +++ b/lib/seeders/reports/assistant_conversation_creator.rb @@ -152,7 +152,7 @@ class Seeders::Reports::AssistantConversationCreator mark_resolved(conversation, resolved_at) travel_to(resolved_at) do trigger_event('conversation_resolved', conversation) - trigger_event('conversation_captain_inference_resolved', conversation) + trigger_captain_event(Events::Types::CAPTAIN_CONVERSATION_RESOLVED, conversation) end travel_back end @@ -178,7 +178,7 @@ class Seeders::Reports::AssistantConversationCreator end def handoff_to_human(conversation) - trigger_event('conversation_captain_inference_handoff', conversation) + trigger_captain_event(Events::Types::CAPTAIN_CONVERSATION_HANDED_OFF, conversation) end def mark_resolved(conversation, resolved_at) @@ -194,6 +194,11 @@ class Seeders::Reports::AssistantConversationCreator ) end + def trigger_captain_event(name, conversation) + event = Events::Base.new(name, Time.current, { conversation: conversation, source: 'inference' }) + Captain::ReportingEventListener.instance.public_send(event.method_name, event) + end + def trigger_reply_time(message, waiting_since) ReportingEventListener.instance.reply_created( Events::Base.new('reply_created', Time.current, diff --git a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb index 946fda149..922a59c7c 100644 --- a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb +++ b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb @@ -53,6 +53,12 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do expect(conversation.messages.last.content).to eq('Hey, welcome to Captain Specs') end + it 'does not emit captain lifecycle events' do + expect(Captain::ConversationEvents).not_to receive(:response_completed) + + described_class.perform_now(conversation, assistant) + end + it 'keeps the default message history limited to public chat messages' do create( :message, @@ -395,6 +401,33 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(0) end + it 'does not emit a response completed event for a stale response' do + allow(mock_agent_runner_service).to receive(:generate_response) do + create(:message, conversation: conversation, content: 'New context', message_type: :incoming) + { 'response' => 'Stale response', 'handoff_tool_called' => false } + end + + expect(Captain::ConversationEvents).not_to receive(:response_completed) + + described_class.perform_now(conversation, assistant, responding_to_message.id) + end + + it 'emits only the response failed event when generation raises after a newer message arrived' do + allow(mock_agent_runner_service).to receive(:generate_response) do + create(:message, conversation: conversation, content: 'New context', message_type: :incoming) + raise StandardError, 'llm down' + end + + expect(Captain::ConversationEvents).to receive(:response_failed) + .with(conversation: conversation, assistant: assistant, reason: 'standard_error', at: kind_of(Time)) + expect(Captain::ConversationEvents).not_to receive(:handed_off) + + described_class.perform_now(conversation, assistant, responding_to_message.id) + + expect(conversation.messages.outgoing.count).to eq(0) + expect(conversation.reload.status).to eq('pending') + end + it 'keeps the pending response fresh when an email auto reply arrives' do create( :message, @@ -456,6 +489,38 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do account.reload expect(account.usage_limits[:captain][:responses][:consumed]).to eq(1) end + + it 'emits a response completed event' do + expect(Captain::ConversationEvents).to receive(:response_completed) + .with(conversation: conversation, assistant: assistant, message: kind_of(Message), at: kind_of(Time)) + + described_class.perform_now(conversation, assistant) + end + + it 'emits response failed and generation failure handoff events when the runner returns an error response' do + allow(mock_agent_runner_service).to receive(:generate_response).and_return( + { 'response' => 'conversation_handoff', 'reasoning' => 'Error occurred: llm down', 'error' => true, + 'error_reason' => 'standard_error', 'handoff_tool_called' => false } + ) + + expect(Captain::ConversationEvents).to receive(:response_failed) + .with(conversation: conversation, assistant: assistant, reason: 'standard_error', at: kind_of(Time)) + expect(Captain::ConversationEvents).to receive(:handed_off) + .with(conversation: conversation, assistant: assistant, source: 'generation_failure', reason_category: :tool_failure, at: kind_of(Time)) + + described_class.perform_now(conversation, assistant) + end + + it 'emits response failed and generation failure handoff events when generation raises' do + allow(mock_agent_runner_service).to receive(:generate_response).and_raise(StandardError, 'llm down') + + expect(Captain::ConversationEvents).to receive(:response_failed) + .with(conversation: conversation, assistant: assistant, reason: 'standard_error', at: kind_of(Time)) + expect(Captain::ConversationEvents).to receive(:handed_off) + .with(conversation: conversation, assistant: assistant, source: 'generation_failure', reason_category: :tool_failure, at: kind_of(Time)) + + described_class.perform_now(conversation, assistant) + end end context 'when captain_v2 handoff tool fires during agent execution' do diff --git a/spec/enterprise/jobs/captain/inbox_pending_conversations_resolution_job_spec.rb b/spec/enterprise/jobs/captain/inbox_pending_conversations_resolution_job_spec.rb index 857e35214..44f0d7afd 100644 --- a/spec/enterprise/jobs/captain/inbox_pending_conversations_resolution_job_spec.rb +++ b/spec/enterprise/jobs/captain/inbox_pending_conversations_resolution_job_spec.rb @@ -37,6 +37,22 @@ RSpec.describe Captain::InboxPendingConversationsResolutionJob, type: :job do expect(open_conversation.reload.status).to eq('open') end + it 'emits a captain resolved event with the time_based source' do + expect(Captain::ConversationEvents).to receive(:resolved) + .with(conversation: resolvable_pending_conversation, assistant: captain_assistant, source: 'time_based', at: kind_of(Time)) + + described_class.perform_now(inbox) + end + + it 'does not create a captain inference reporting event' do + perform_enqueued_jobs do + described_class.perform_now(inbox) + end + + expect(ReportingEvent.exists?(conversation_id: resolvable_pending_conversation.id, + name: 'conversation_captain_inference_resolved')).to be(false) + end + it 'does not call ConversationCompletionService' do allow(Captain::ConversationCompletionService).to receive(:new) @@ -160,6 +176,13 @@ RSpec.describe Captain::InboxPendingConversationsResolutionJob, type: :job do ) end + it 'emits a captain resolved event with the inference source' do + expect(Captain::ConversationEvents).to receive(:resolved) + .with(conversation: resolvable_pending_conversation, assistant: captain_assistant, source: 'inference', at: kind_of(Time)) + + described_class.perform_now(inbox) + end + it 'creates a captain inference resolved reporting event' do perform_enqueued_jobs do described_class.perform_now(inbox) @@ -259,6 +282,14 @@ RSpec.describe Captain::InboxPendingConversationsResolutionJob, type: :job do ) end + it 'emits a captain handoff event with the inference source' do + expect(Captain::ConversationEvents).to receive(:handed_off) + .with(conversation: resolvable_pending_conversation, assistant: captain_assistant, source: 'inference', + reason_category: :pending_clarification, at: kind_of(Time)) + + described_class.perform_now(inbox) + end + it 'creates a captain inference handoff reporting event' do perform_enqueued_jobs do described_class.perform_now(inbox) diff --git a/spec/enterprise/lib/captain/tools/handoff_tool_spec.rb b/spec/enterprise/lib/captain/tools/handoff_tool_spec.rb index 2047a2da1..d928e7d8f 100644 --- a/spec/enterprise/lib/captain/tools/handoff_tool_spec.rb +++ b/spec/enterprise/lib/captain/tools/handoff_tool_spec.rb @@ -93,6 +93,21 @@ RSpec.describe Captain::Tools::HandoffTool, type: :model do end.not_to change(Message, :count) expect(conversation.reload.status).to eq('pending') end + + it 'emits a captain handoff event with the tool source after the locked handoff completes' do + expect(Captain::ConversationEvents).to receive(:handed_off) + .with(conversation: conversation, assistant: assistant, source: 'tool', at: kind_of(Time)) + + tool.perform(tool_context, reason: 'Customer needs specialized support') + end + + it 'does not emit a captain handoff event when the handoff is skipped as stale' do + create(:message, conversation: conversation, account: account, inbox: inbox, message_type: :incoming) + + expect(Captain::ConversationEvents).not_to receive(:handed_off) + + tool.perform(tool_context, reason: 'Customer needs specialized support') + end end context 'with Captain V1' do @@ -155,6 +170,13 @@ RSpec.describe Captain::Tools::HandoffTool, type: :model do tool.perform(tool_context, reason: 'Test reason') end + it 'emits a captain handoff event with the tool source' do + expect(Captain::ConversationEvents).to receive(:handed_off) + .with(conversation: conversation, assistant: assistant, source: 'tool', at: kind_of(Time)) + + tool.perform(tool_context, reason: 'Test reason') + end + it 'creates a conversation_bot_handoff reporting event' do create(:captain_inbox, captain_assistant: assistant, inbox: inbox) Current.executed_by = assistant diff --git a/spec/enterprise/lib/captain/tools/resolve_conversation_tool_spec.rb b/spec/enterprise/lib/captain/tools/resolve_conversation_tool_spec.rb index 2eae2905a..e3054230c 100644 --- a/spec/enterprise/lib/captain/tools/resolve_conversation_tool_spec.rb +++ b/spec/enterprise/lib/captain/tools/resolve_conversation_tool_spec.rb @@ -29,6 +29,13 @@ RSpec.describe Captain::Tools::ResolveConversationTool do ) end + it 'emits a captain resolution event with the tool source' do + expect(Captain::ConversationEvents).to receive(:resolved) + .with(conversation: conversation, assistant: assistant, source: 'tool', at: kind_of(Time)) + + tool.perform(tool_context, reason: 'Possible spam') + end + it 'creates a conversation_resolved reporting event' do create(:captain_inbox, captain_assistant: assistant, inbox: inbox) @@ -65,6 +72,12 @@ RSpec.describe Captain::Tools::ResolveConversationTool do describe 'resolving an already resolved conversation' do let(:conversation) { create(:conversation, account: account, inbox: inbox, status: :resolved) } + it 'does not emit a captain resolution event' do + expect(Captain::ConversationEvents).not_to receive(:resolved) + + tool.perform(tool_context, reason: 'Possible spam') + end + it 'does not re-resolve and returns an already resolved message' do queue_adapter = ActiveJob::Base.queue_adapter queue_adapter.enqueued_jobs.clear diff --git a/spec/enterprise/listeners/captain/reporting_event_listener_spec.rb b/spec/enterprise/listeners/captain/reporting_event_listener_spec.rb new file mode 100644 index 000000000..8b14a39ef --- /dev/null +++ b/spec/enterprise/listeners/captain/reporting_event_listener_spec.rb @@ -0,0 +1,53 @@ +require 'rails_helper' + +RSpec.describe Captain::ReportingEventListener do + let(:listener) { described_class.instance } + let(:account) { create(:account) } + let(:inbox) { create(:inbox, account: account) } + let(:conversation) { create(:conversation, account: account, inbox: inbox) } + let(:assistant) { create(:captain_assistant, account: account) } + + describe '#captain_conversation_resolved' do + it 'creates a captain inference resolved reporting event for inference resolutions' do + decision_time = conversation.created_at + 60.seconds + event = Events::Base.new( + Events::Types::CAPTAIN_CONVERSATION_RESOLVED, decision_time, + conversation: conversation, assistant: assistant, source: 'inference' + ) + + listener.captain_conversation_resolved(event) + + reporting_event = account.reporting_events.where(name: 'conversation_captain_inference_resolved').first + expect(reporting_event).to be_present + expect(reporting_event.value).to eq 60 + expect(reporting_event.event_end_time).to be_within(1.second).of(decision_time) + end + end + + describe '#captain_conversation_handed_off' do + it 'creates a captain inference handoff reporting event for inference handoffs' do + decision_time = conversation.created_at + 90.seconds + event = Events::Base.new( + Events::Types::CAPTAIN_CONVERSATION_HANDED_OFF, decision_time, + conversation: conversation, assistant: assistant, source: 'inference', reason_category: :pending_clarification + ) + + listener.captain_conversation_handed_off(event) + + reporting_event = account.reporting_events.where(name: 'conversation_captain_inference_handoff').first + expect(reporting_event).to be_present + expect(reporting_event.value).to eq 90 + expect(reporting_event.event_end_time).to be_within(1.second).of(decision_time) + end + + it 'does not create a reporting event for non-inference handoffs' do + event = Events::Base.new( + Events::Types::CAPTAIN_CONVERSATION_HANDED_OFF, Time.zone.now, + conversation: conversation, assistant: assistant, source: 'tool', reason_category: nil + ) + + expect { listener.captain_conversation_handed_off(event) } + .not_to(change { account.reporting_events.count }) + end + end +end diff --git a/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb b/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb index 304a7dd34..494d60dbc 100644 --- a/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb +++ b/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb @@ -259,6 +259,8 @@ RSpec.describe Captain::Assistant::AgentRunnerService do expect(result).to eq({ 'response' => 'conversation_handoff', 'reasoning' => 'Error occurred: Test error', + 'error' => true, + 'error_reason' => 'standard_error', 'handoff_tool_called' => false }) end @@ -287,6 +289,8 @@ RSpec.describe Captain::Assistant::AgentRunnerService do expect(result).to eq({ 'response' => 'conversation_handoff', 'reasoning' => 'Error occurred: Test error', + 'error' => true, + 'error_reason' => 'standard_error', 'handoff_tool_called' => false }) end @@ -311,6 +315,8 @@ RSpec.describe Captain::Assistant::AgentRunnerService do expect(result).to eq({ 'response' => 'conversation_handoff', 'reasoning' => 'Error occurred: Test error', + 'error' => true, + 'error_reason' => 'standard_error', 'handoff_tool_called' => true }) end diff --git a/spec/enterprise/services/captain/conversation_events_spec.rb b/spec/enterprise/services/captain/conversation_events_spec.rb new file mode 100644 index 000000000..b6f22d6cb --- /dev/null +++ b/spec/enterprise/services/captain/conversation_events_spec.rb @@ -0,0 +1,88 @@ +require 'rails_helper' + +RSpec.describe Captain::ConversationEvents do + let(:account) { create(:account) } + let(:conversation) { create(:conversation, account: account) } + let(:assistant) { create(:captain_assistant, account: account) } + let(:timestamp) { Time.zone.now } + + describe '.engaged' do + it 'dispatches the engagement event with normalized context' do + expect(Rails.configuration.dispatcher).to receive(:dispatch).with( + Events::Types::CAPTAIN_CONVERSATION_ENGAGED, + timestamp, + { conversation: conversation, assistant: assistant } + ) + + described_class.engaged(conversation: conversation, assistant: assistant, at: timestamp) + end + end + + describe '.handed_off' do + it 'dispatches the handoff event with source and reason category' do + expect(Rails.configuration.dispatcher).to receive(:dispatch).with( + Events::Types::CAPTAIN_CONVERSATION_HANDED_OFF, + timestamp, + { conversation: conversation, assistant: assistant, source: 'inference', reason_category: :pending_clarification } + ) + + described_class.handed_off( + conversation: conversation, assistant: assistant, source: 'inference', reason_category: :pending_clarification, at: timestamp + ) + end + end + + describe '.resolved' do + it 'dispatches the resolution event with source' do + expect(Rails.configuration.dispatcher).to receive(:dispatch).with( + Events::Types::CAPTAIN_CONVERSATION_RESOLVED, + timestamp, + { conversation: conversation, assistant: assistant, source: 'inference' } + ) + + described_class.resolved(conversation: conversation, assistant: assistant, source: 'inference', at: timestamp) + end + end + + describe '.response_completed' do + it 'dispatches the response completed event with the result message' do + message = create(:message, conversation: conversation, account: account) + + expect(Rails.configuration.dispatcher).to receive(:dispatch).with( + Events::Types::CAPTAIN_RESPONSE_COMPLETED, + timestamp, + { conversation: conversation, assistant: assistant, message: message } + ) + + described_class.response_completed(conversation: conversation, assistant: assistant, message: message, at: timestamp) + end + end + + describe 'when dispatch fails' do + it 'captures the exception instead of raising into the Captain flow' do + conversation + assistant + error = StandardError.new('redis down') + allow(Rails.configuration.dispatcher).to receive(:dispatch).and_raise(error) + expect(ChatwootExceptionTracker).to receive(:new) + .with(error, account: account) + .and_return(instance_double(ChatwootExceptionTracker, capture_exception: true)) + + expect do + described_class.engaged(conversation: conversation, assistant: assistant, at: timestamp) + end.not_to raise_error + end + end + + describe '.response_failed' do + it 'dispatches the response failed event with the failure reason' do + expect(Rails.configuration.dispatcher).to receive(:dispatch).with( + Events::Types::CAPTAIN_RESPONSE_FAILED, + timestamp, + { conversation: conversation, assistant: assistant, reason: 'faraday_bad_request_error' } + ) + + described_class.response_failed(conversation: conversation, assistant: assistant, reason: 'faraday_bad_request_error', at: timestamp) + end + end +end diff --git a/spec/enterprise/services/enterprise/message_templates/hook_execution_service_spec.rb b/spec/enterprise/services/enterprise/message_templates/hook_execution_service_spec.rb index 94894dd6e..b545cd91e 100644 --- a/spec/enterprise/services/enterprise/message_templates/hook_execution_service_spec.rb +++ b/spec/enterprise/services/enterprise/message_templates/hook_execution_service_spec.rb @@ -136,6 +136,21 @@ RSpec.describe MessageTemplates::HookExecutionService do create(:message, conversation: conversation, message_type: :incoming, account: account) end + + it 'emits the engagement event when captain V2 is enabled' do + account.enable_features!('captain_integration_v2') + + expect(Captain::ConversationEvents).to receive(:engaged) + .with(conversation: conversation, assistant: assistant, at: kind_of(Time)) + + create(:message, conversation: conversation, message_type: :incoming, account: account) + end + + it 'does not emit the engagement event when captain V2 is disabled' do + expect(Captain::ConversationEvents).not_to receive(:engaged) + + create(:message, conversation: conversation, message_type: :incoming, account: account) + end end context 'when captain quota is exceeded within business hours' do @@ -157,6 +172,21 @@ RSpec.describe MessageTemplates::HookExecutionService do expect(conversation.reload.status).to eq('open') end + + it 'emits a usage limit handoff event when captain V2 is enabled' do + account.enable_features!('captain_integration_v2') + + expect(Captain::ConversationEvents).to receive(:handed_off) + .with(conversation: conversation, assistant: assistant, source: 'usage_limit', reason_category: :usage_limit, at: kind_of(Time)) + + create(:message, conversation: conversation, message_type: :incoming, account: account) + end + + it 'does not emit a handoff event when captain V2 is disabled' do + expect(Captain::ConversationEvents).not_to receive(:handed_off) + + create(:message, conversation: conversation, message_type: :incoming, account: account) + end end end diff --git a/spec/listeners/reporting_event_listener_spec.rb b/spec/listeners/reporting_event_listener_spec.rb index a497d7dd8..968baa481 100644 --- a/spec/listeners/reporting_event_listener_spec.rb +++ b/spec/listeners/reporting_event_listener_spec.rb @@ -309,38 +309,6 @@ describe ReportingEventListener do end end - describe '#conversation_captain_inference_resolved' do - it 'creates conversation_captain_inference_resolved event' do - expect(account.reporting_events.where(name: 'conversation_captain_inference_resolved').count).to be 0 - decision_time = conversation.created_at + 60.seconds - event = Events::Base.new('conversation.captain_inference_resolved', decision_time, conversation: conversation) - allow(conversation).to receive(:updated_at).and_return(decision_time + 5.minutes) - - listener.conversation_captain_inference_resolved(event) - - reporting_event = account.reporting_events.where(name: 'conversation_captain_inference_resolved').first - expect(reporting_event).to be_present - expect(reporting_event.value).to eq 60 - expect(reporting_event.event_end_time).to be_within(1.second).of(decision_time) - end - end - - describe '#conversation_captain_inference_handoff' do - it 'creates conversation_captain_inference_handoff event' do - expect(account.reporting_events.where(name: 'conversation_captain_inference_handoff').count).to be 0 - decision_time = conversation.created_at + 90.seconds - event = Events::Base.new('conversation.captain_inference_handoff', decision_time, conversation: conversation) - allow(conversation).to receive(:updated_at).and_return(decision_time + 5.minutes) - - listener.conversation_captain_inference_handoff(event) - - reporting_event = account.reporting_events.where(name: 'conversation_captain_inference_handoff').first - expect(reporting_event).to be_present - expect(reporting_event.value).to eq 90 - expect(reporting_event.event_end_time).to be_within(1.second).of(decision_time) - end - end - describe '#conversation_opened' do context 'when conversation is opened for the first time' do let(:new_conversation) { create(:conversation, account: account, inbox: inbox, assignee: user) }