diff --git a/app/listeners/reporting_event_listener.rb b/app/listeners/reporting_event_listener.rb index 40aa0b0ac..d8137e727 100644 --- a/app/listeners/reporting_event_listener.rb +++ b/app/listeners/reporting_event_listener.rb @@ -98,14 +98,6 @@ class ReportingEventListener < BaseListener safe_rollup(reporting_event) end - def conversation_captain_inference_resolved(event) - create_captain_inference_event(event, 'conversation_captain_inference_resolved') - end - - def conversation_captain_inference_handoff(event) - create_captain_inference_event(event, 'conversation_captain_inference_handoff') - end - def conversation_opened(event) conversation = extract_conversation_and_account(event)[0] event_end_time = event.timestamp @@ -148,22 +140,6 @@ class ReportingEventListener < BaseListener reporting_event.save! end - def create_captain_inference_event(event, event_name) - conversation = extract_conversation_and_account(event)[0] - time_to_event = event.timestamp.to_i - conversation.created_at.to_i - - ReportingEvent.create!( - name: event_name, - value: time_to_event, - account_id: conversation.account_id, - inbox_id: conversation.inbox_id, - user_id: conversation.assignee_id, - conversation_id: conversation.id, - event_start_time: conversation.created_at, - event_end_time: event.timestamp - ) - end - def create_bot_resolved_event(conversation, reporting_event) return unless conversation.inbox.active_bot? # We don't want to create a bot_resolved event if there is user interaction on the conversation diff --git a/enterprise/app/dispatchers/enterprise/async_dispatcher.rb b/enterprise/app/dispatchers/enterprise/async_dispatcher.rb index 0225e2d04..7fe65143e 100644 --- a/enterprise/app/dispatchers/enterprise/async_dispatcher.rb +++ b/enterprise/app/dispatchers/enterprise/async_dispatcher.rb @@ -1,7 +1,8 @@ module Enterprise::AsyncDispatcher def listeners super + [ - CaptainListener.instance + CaptainListener.instance, + Captain::ReportingEventListener.instance ] end end diff --git a/enterprise/app/jobs/captain/conversation/response_builder_job.rb b/enterprise/app/jobs/captain/conversation/response_builder_job.rb index b86f859f3..f37086816 100644 --- a/enterprise/app/jobs/captain/conversation/response_builder_job.rb +++ b/enterprise/app/jobs/captain/conversation/response_builder_job.rb @@ -1,6 +1,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob include Captain::Conversation::V1ActionClassifier include Captain::Conversation::V1FalsePromiseHandler + include Captain::Conversation::V2LifecycleEvents include Captain::Conversation::MessageBuilder MAX_MESSAGE_LENGTH = 10_000 @@ -64,20 +65,30 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob end def process_response + # The V2 runner rescues its own generation errors and signals them via an error + # response instead of raising, so the failure event must be emitted here — the + # top-level handle_error path only sees exceptions raised outside the runner. + record_v2_response_failure(@response['error_reason']) if v2_generation_errored? + if v2_handoff_tool_fired? process_v2_handoff_response elsif v1_handoff_requested? - # V1 only signals via the response string — no state has been touched yet. If - # the conversation isn't pending anymore, a human took over mid-run; bail out - # rather than posting a stale handoff message on top of their reply. - return unless conversation_pending? - - process_v1_handoff + process_v1_handoff_request elsif conversation_pending? process_standard_response end end + def process_v1_handoff_request + # V1 only signals via the response string — no state has been touched yet. If + # the conversation isn't pending anymore, a human took over mid-run; bail out + # rather than posting a stale handoff message on top of their reply. + return unless conversation_pending? + + process_v1_handoff + record_v2_failure_handoff if v2_generation_errored? + end + def process_standard_response message = nil ActiveRecord::Base.transaction do @@ -90,6 +101,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob return unless message capture_assistant_session(result_message: message, credits_consumed: 1.0) + record_v2_response_completed(message) if captain_v2_enabled? end def process_v2_handoff_response @@ -98,6 +110,10 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob if captain_v2_enabled? return unless v2_handoff_tool_completed? || conversation_pending? + # Known gap, accepted for now: the fallback V1 handoff (tool fired but never + # completed) emits no captain.conversation.handed_off event — the tool emits + # only after a successful bot_handoff!. If outcome data ever needs it, emit + # here with a distinct source such as 'tool_fallback'. v2_handoff_tool_completed? ? process_v2_handoff : process_v1_handoff else conversation_pending? ? process_v1_handoff : process_v2_handoff @@ -173,10 +189,19 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob @response ||= {} @response['action_source'] ||= 'error' @response['action_reason'] ||= error_action_reason(error) - process_v1_handoff if conversation_pending? && (!captain_v2_enabled? || !newer_customer_message_arrived?) + record_v2_response_failure(error_action_reason(error)) if captain_v2_enabled? + process_error_handoff true end + def process_error_handoff + return unless conversation_pending? + return if captain_v2_enabled? && newer_customer_message_arrived? + + process_v1_handoff + record_v2_failure_handoff if captain_v2_enabled? + end + def log_error(error) ChatwootExceptionTracker.new(error, account: account).capture_exception end diff --git a/enterprise/app/jobs/captain/conversation/v2_lifecycle_events.rb b/enterprise/app/jobs/captain/conversation/v2_lifecycle_events.rb new file mode 100644 index 000000000..37501e9ff --- /dev/null +++ b/enterprise/app/jobs/captain/conversation/v2_lifecycle_events.rb @@ -0,0 +1,30 @@ +module Captain::Conversation::V2LifecycleEvents + private + + def v2_generation_errored? + captain_v2_enabled? && @response['error'].present? + end + + def record_v2_response_completed(message) + Captain::ConversationEvents.response_completed( + conversation: @conversation, + assistant: @assistant, + message: message, + at: Time.current + ) + end + + def record_v2_response_failure(reason) + Captain::ConversationEvents.response_failed(conversation: @conversation, assistant: @assistant, reason: reason, at: Time.current) + end + + def record_v2_failure_handoff + Captain::ConversationEvents.handed_off( + conversation: @conversation, + assistant: @assistant, + source: Captain::ConversationEvents::Sources::GENERATION_FAILURE, + reason_category: :tool_failure, + at: Time.current + ) + end +end diff --git a/enterprise/app/jobs/captain/inbox_pending_conversations_resolution_job.rb b/enterprise/app/jobs/captain/inbox_pending_conversations_resolution_job.rb index 0be179f04..958c809bf 100644 --- a/enterprise/app/jobs/captain/inbox_pending_conversations_resolution_job.rb +++ b/enterprise/app/jobs/captain/inbox_pending_conversations_resolution_job.rb @@ -28,6 +28,12 @@ class Captain::InboxPendingConversationsResolutionJob < ApplicationJob resolvable_pending_conversations(inbox).each do |conversation| create_resolution_message(conversation, inbox) conversation.resolved! + Captain::ConversationEvents.resolved( + conversation: conversation, + assistant: inbox.captain_assistant, + source: Captain::ConversationEvents::Sources::TIME_BASED, + at: Time.current + ) end end @@ -77,7 +83,12 @@ class Captain::InboxPendingConversationsResolutionJob < ApplicationJob reason: CAPTAIN_INFERENCE_RESOLVE_ACTIVITY_REASON, reason_type: :inference ) { conversation.resolved! } - conversation.dispatch_captain_inference_resolved_event + Captain::ConversationEvents.resolved( + conversation: conversation, + assistant: inbox.captain_assistant, + source: Captain::ConversationEvents::Sources::INFERENCE, + at: Time.current + ) end def handoff_conversation(conversation, inbox, reason) @@ -87,7 +98,13 @@ class Captain::InboxPendingConversationsResolutionJob < ApplicationJob reason: CAPTAIN_INFERENCE_HANDOFF_ACTIVITY_REASON, reason_type: :inference ) { conversation.bot_handoff! } - conversation.dispatch_captain_inference_handoff_event + Captain::ConversationEvents.handed_off( + conversation: conversation, + assistant: inbox.captain_assistant, + source: Captain::ConversationEvents::Sources::INFERENCE, + reason_category: :pending_clarification, + at: Time.current + ) send_out_of_office_message_if_applicable(conversation.reload) end diff --git a/enterprise/app/listeners/captain/reporting_event_listener.rb b/enterprise/app/listeners/captain/reporting_event_listener.rb new file mode 100644 index 000000000..6033dca45 --- /dev/null +++ b/enterprise/app/listeners/captain/reporting_event_listener.rb @@ -0,0 +1,27 @@ +class Captain::ReportingEventListener < BaseListener + def captain_conversation_handed_off(event) + create_captain_inference_event(event, 'conversation_captain_inference_handoff') if event.data[:source] == 'inference' + end + + def captain_conversation_resolved(event) + create_captain_inference_event(event, 'conversation_captain_inference_resolved') if event.data[:source] == 'inference' + end + + private + + def create_captain_inference_event(event, event_name) + conversation = extract_conversation_and_account(event)[0] + time_to_event = event.timestamp.to_i - conversation.created_at.to_i + + ReportingEvent.create!( + name: event_name, + value: time_to_event, + account_id: conversation.account_id, + inbox_id: conversation.inbox_id, + user_id: conversation.assignee_id, + conversation_id: conversation.id, + event_start_time: conversation.created_at, + event_end_time: event.timestamp + ) + end +end diff --git a/enterprise/app/models/enterprise/conversation.rb b/enterprise/app/models/enterprise/conversation.rb index 0765e7279..f57aa3f05 100644 --- a/enterprise/app/models/enterprise/conversation.rb +++ b/enterprise/app/models/enterprise/conversation.rb @@ -1,14 +1,6 @@ module Enterprise::Conversation attr_accessor :captain_activity_reason, :captain_activity_reason_type - def dispatch_captain_inference_resolved_event - dispatch_captain_inference_event(Events::Types::CONVERSATION_CAPTAIN_INFERENCE_RESOLVED) - end - - def dispatch_captain_inference_handoff_event - dispatch_captain_inference_event(Events::Types::CONVERSATION_CAPTAIN_INFERENCE_HANDOFF) - end - def list_of_keys super + %w[sla_policy_id] end @@ -50,10 +42,6 @@ module Enterprise::Conversation current_applied_sla.update!(completed_at: resolved? ? Time.current : nil) end - def dispatch_captain_inference_event(event_name) - dispatcher_dispatch(event_name) - end - def call_attributes_changed? return false if previous_changes['additional_attributes'].blank? diff --git a/enterprise/app/services/captain/assistant/agent_runner_service.rb b/enterprise/app/services/captain/assistant/agent_runner_service.rb index 0d3d52e86..b2c7efa45 100644 --- a/enterprise/app/services/captain/assistant/agent_runner_service.rb +++ b/enterprise/app/services/captain/assistant/agent_runner_service.rb @@ -34,7 +34,7 @@ class Captain::Assistant::AgentRunnerService Rails.logger.error "[Captain V2] AgentRunnerService error: #{e.message}" Rails.logger.error e.backtrace.join("\n") - error_response(e.message) + error_response(e) end def response_discarded? = @response_discarded == true @@ -132,10 +132,12 @@ class Captain::Assistant::AgentRunnerService @message_length_limit ||= Captain::MessageLengthLimit.for(@conversation) end - def error_response(error_message) + def error_response(error) { 'response' => 'conversation_handoff', - 'reasoning' => "Error occurred: #{error_message}", + 'reasoning' => "Error occurred: #{error.message}", + 'error' => true, + 'error_reason' => error.class.name.underscore.tr('/', '_'), 'handoff_tool_called' => @handoff_tool_called } end diff --git a/enterprise/app/services/captain/conversation_events.rb b/enterprise/app/services/captain/conversation_events.rb new file mode 100644 index 000000000..964b2d365 --- /dev/null +++ b/enterprise/app/services/captain/conversation_events.rb @@ -0,0 +1,72 @@ +class Captain::ConversationEvents + module Sources + TOOL = 'tool'.freeze + GENERATION_FAILURE = 'generation_failure'.freeze + TIME_BASED = 'time_based'.freeze + INFERENCE = 'inference'.freeze + USAGE_LIMIT = 'usage_limit'.freeze + end + + class << self + def engaged(conversation:, assistant:, at:) + dispatch( + Events::Types::CAPTAIN_CONVERSATION_ENGAGED, + at: at, + conversation: conversation, + assistant: assistant + ) + end + + def handed_off(conversation:, assistant:, source:, at:, reason_category: nil) + dispatch( + Events::Types::CAPTAIN_CONVERSATION_HANDED_OFF, + at: at, + conversation: conversation, + assistant: assistant, + source: source, + reason_category: reason_category + ) + end + + def resolved(conversation:, assistant:, source:, at:) + dispatch( + Events::Types::CAPTAIN_CONVERSATION_RESOLVED, + at: at, + conversation: conversation, + assistant: assistant, + source: source + ) + end + + def response_completed(conversation:, assistant:, message:, at:) + dispatch( + Events::Types::CAPTAIN_RESPONSE_COMPLETED, + at: at, + conversation: conversation, + assistant: assistant, + message: message + ) + end + + def response_failed(conversation:, assistant:, reason:, at:) + dispatch( + Events::Types::CAPTAIN_RESPONSE_FAILED, + at: at, + conversation: conversation, + assistant: assistant, + reason: reason + ) + end + + private + + # Lifecycle events are secondary effects: a dispatch failure must never alter + # the customer-facing Captain flow (blocking a response from being scheduled, + # or turning an already-delivered reply into a spurious handoff). + def dispatch(event_name, at:, **data) + Rails.configuration.dispatcher.dispatch(event_name, at, data) + rescue StandardError => e + ChatwootExceptionTracker.new(e, account: data[:conversation]&.account).capture_exception + end + end +end diff --git a/enterprise/app/services/enterprise/message_templates/hook_execution_service.rb b/enterprise/app/services/enterprise/message_templates/hook_execution_service.rb index d32968384..b2491607e 100644 --- a/enterprise/app/services/enterprise/message_templates/hook_execution_service.rb +++ b/enterprise/app/services/enterprise/message_templates/hook_execution_service.rb @@ -6,6 +6,7 @@ module Enterprise::MessageTemplates::HookExecutionService return unless should_process_captain_response? return perform_handoff unless inbox.captain_active? + track_captain_engagement schedule_captain_response end @@ -29,6 +30,20 @@ module Enterprise::MessageTemplates::HookExecutionService private + def track_captain_engagement + return unless captain_v2_enabled? + + Captain::ConversationEvents.engaged( + conversation: conversation, + assistant: inbox.captain_assistant, + at: message.created_at + ) + end + + def captain_v2_enabled? + conversation.account.feature_enabled?('captain_integration_v2') + end + def schedule_captain_response job_args = [conversation, conversation.inbox.captain_assistant] captain_v2_enabled = conversation.account.feature_enabled?('captain_integration_v2') @@ -81,6 +96,15 @@ module Enterprise::MessageTemplates::HookExecutionService content: 'Transferring to another agent for further assistance.' ) conversation.bot_handoff! + if captain_v2_enabled? + Captain::ConversationEvents.handed_off( + conversation: conversation, + assistant: inbox.captain_assistant, + source: Captain::ConversationEvents::Sources::USAGE_LIMIT, + reason_category: :usage_limit, + at: Time.current + ) + end send_out_of_office_message_after_handoff end diff --git a/enterprise/lib/captain/tools/handoff_tool.rb b/enterprise/lib/captain/tools/handoff_tool.rb index 73a76c2d0..e1d8a11c8 100644 --- a/enterprise/lib/captain/tools/handoff_tool.rb +++ b/enterprise/lib/captain/tools/handoff_tool.rb @@ -54,6 +54,7 @@ class Captain::Tools::HandoffTool < Captain::Tools::BasePublicTool tool_context.state[:captain_v2_handoff_tool_completed] = true # Queue the event after the state change commits so notification jobs always see the open conversation. conversation.dispatch_bot_handoff_event + emit_tool_handoff_event(conversation) # Send out of office message if applicable (since template messages were suppressed while Captain was handling) send_out_of_office_message_if_applicable(conversation) @@ -67,10 +68,16 @@ class Captain::Tools::HandoffTool < Captain::Tools::BasePublicTool ) record_handoff_note(tool_context, note) if reason.present? conversation.bot_handoff! + emit_tool_handoff_event(conversation) send_out_of_office_message_if_applicable(conversation) :completed end + def emit_tool_handoff_event(conversation) + Captain::ConversationEvents.handed_off(conversation: conversation, assistant: @assistant, + source: Captain::ConversationEvents::Sources::TOOL, at: Time.current) + end + def record_handoff_note(tool_context, note) metadata = tool_context.state[:cw_metadata] ||= {} metadata[:handoff_note_id] = note.id diff --git a/enterprise/lib/captain/tools/resolve_conversation_tool.rb b/enterprise/lib/captain/tools/resolve_conversation_tool.rb index eeffc3f06..ba4d8323a 100644 --- a/enterprise/lib/captain/tools/resolve_conversation_tool.rb +++ b/enterprise/lib/captain/tools/resolve_conversation_tool.rb @@ -11,6 +11,8 @@ class Captain::Tools::ResolveConversationTool < Captain::Tools::BasePublicTool log_tool_usage('resolve_conversation', { conversation_id: conversation.id, reason: reason }) conversation.with_captain_activity_context(reason: reason, reason_type: :tool) { conversation.resolved! } + Captain::ConversationEvents.resolved(conversation: conversation, assistant: @assistant, + source: Captain::ConversationEvents::Sources::TOOL, at: Time.current) "Conversation ##{conversation.display_id} resolved#{" (Reason: #{reason})" if reason}" end diff --git a/lib/events/types.rb b/lib/events/types.rb index 171649a7d..653739826 100644 --- a/lib/events/types.rb +++ b/lib/events/types.rb @@ -22,8 +22,11 @@ module Events::Types # FIXME: deprecate the opened and resolved events in future in favor of status changed event. CONVERSATION_OPENED = 'conversation.opened' CONVERSATION_RESOLVED = 'conversation.resolved' - CONVERSATION_CAPTAIN_INFERENCE_RESOLVED = 'conversation.captain_inference_resolved' - CONVERSATION_CAPTAIN_INFERENCE_HANDOFF = 'conversation.captain_inference_handoff' + CAPTAIN_CONVERSATION_ENGAGED = 'captain.conversation.engaged' + CAPTAIN_CONVERSATION_HANDED_OFF = 'captain.conversation.handed_off' + CAPTAIN_CONVERSATION_RESOLVED = 'captain.conversation.resolved' + CAPTAIN_RESPONSE_COMPLETED = 'captain.response.completed' + CAPTAIN_RESPONSE_FAILED = 'captain.response.failed' CONVERSATION_STATUS_CHANGED = 'conversation.status_changed' CONVERSATION_CONTACT_CHANGED = 'conversation.contact_changed' diff --git a/lib/seeders/reports/assistant_conversation_creator.rb b/lib/seeders/reports/assistant_conversation_creator.rb index 9462d899e..d21e05976 100644 --- a/lib/seeders/reports/assistant_conversation_creator.rb +++ b/lib/seeders/reports/assistant_conversation_creator.rb @@ -152,7 +152,7 @@ class Seeders::Reports::AssistantConversationCreator mark_resolved(conversation, resolved_at) travel_to(resolved_at) do trigger_event('conversation_resolved', conversation) - trigger_event('conversation_captain_inference_resolved', conversation) + trigger_captain_event(Events::Types::CAPTAIN_CONVERSATION_RESOLVED, conversation) end travel_back end @@ -178,7 +178,7 @@ class Seeders::Reports::AssistantConversationCreator end def handoff_to_human(conversation) - trigger_event('conversation_captain_inference_handoff', conversation) + trigger_captain_event(Events::Types::CAPTAIN_CONVERSATION_HANDED_OFF, conversation) end def mark_resolved(conversation, resolved_at) @@ -194,6 +194,11 @@ class Seeders::Reports::AssistantConversationCreator ) end + def trigger_captain_event(name, conversation) + event = Events::Base.new(name, Time.current, { conversation: conversation, source: 'inference' }) + Captain::ReportingEventListener.instance.public_send(event.method_name, event) + end + def trigger_reply_time(message, waiting_since) ReportingEventListener.instance.reply_created( Events::Base.new('reply_created', Time.current, diff --git a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb index 946fda149..922a59c7c 100644 --- a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb +++ b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb @@ -53,6 +53,12 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do expect(conversation.messages.last.content).to eq('Hey, welcome to Captain Specs') end + it 'does not emit captain lifecycle events' do + expect(Captain::ConversationEvents).not_to receive(:response_completed) + + described_class.perform_now(conversation, assistant) + end + it 'keeps the default message history limited to public chat messages' do create( :message, @@ -395,6 +401,33 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(0) end + it 'does not emit a response completed event for a stale response' do + allow(mock_agent_runner_service).to receive(:generate_response) do + create(:message, conversation: conversation, content: 'New context', message_type: :incoming) + { 'response' => 'Stale response', 'handoff_tool_called' => false } + end + + expect(Captain::ConversationEvents).not_to receive(:response_completed) + + described_class.perform_now(conversation, assistant, responding_to_message.id) + end + + it 'emits only the response failed event when generation raises after a newer message arrived' do + allow(mock_agent_runner_service).to receive(:generate_response) do + create(:message, conversation: conversation, content: 'New context', message_type: :incoming) + raise StandardError, 'llm down' + end + + expect(Captain::ConversationEvents).to receive(:response_failed) + .with(conversation: conversation, assistant: assistant, reason: 'standard_error', at: kind_of(Time)) + expect(Captain::ConversationEvents).not_to receive(:handed_off) + + described_class.perform_now(conversation, assistant, responding_to_message.id) + + expect(conversation.messages.outgoing.count).to eq(0) + expect(conversation.reload.status).to eq('pending') + end + it 'keeps the pending response fresh when an email auto reply arrives' do create( :message, @@ -456,6 +489,38 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do account.reload expect(account.usage_limits[:captain][:responses][:consumed]).to eq(1) end + + it 'emits a response completed event' do + expect(Captain::ConversationEvents).to receive(:response_completed) + .with(conversation: conversation, assistant: assistant, message: kind_of(Message), at: kind_of(Time)) + + described_class.perform_now(conversation, assistant) + end + + it 'emits response failed and generation failure handoff events when the runner returns an error response' do + allow(mock_agent_runner_service).to receive(:generate_response).and_return( + { 'response' => 'conversation_handoff', 'reasoning' => 'Error occurred: llm down', 'error' => true, + 'error_reason' => 'standard_error', 'handoff_tool_called' => false } + ) + + expect(Captain::ConversationEvents).to receive(:response_failed) + .with(conversation: conversation, assistant: assistant, reason: 'standard_error', at: kind_of(Time)) + expect(Captain::ConversationEvents).to receive(:handed_off) + .with(conversation: conversation, assistant: assistant, source: 'generation_failure', reason_category: :tool_failure, at: kind_of(Time)) + + described_class.perform_now(conversation, assistant) + end + + it 'emits response failed and generation failure handoff events when generation raises' do + allow(mock_agent_runner_service).to receive(:generate_response).and_raise(StandardError, 'llm down') + + expect(Captain::ConversationEvents).to receive(:response_failed) + .with(conversation: conversation, assistant: assistant, reason: 'standard_error', at: kind_of(Time)) + expect(Captain::ConversationEvents).to receive(:handed_off) + .with(conversation: conversation, assistant: assistant, source: 'generation_failure', reason_category: :tool_failure, at: kind_of(Time)) + + described_class.perform_now(conversation, assistant) + end end context 'when captain_v2 handoff tool fires during agent execution' do diff --git a/spec/enterprise/jobs/captain/inbox_pending_conversations_resolution_job_spec.rb b/spec/enterprise/jobs/captain/inbox_pending_conversations_resolution_job_spec.rb index 857e35214..44f0d7afd 100644 --- a/spec/enterprise/jobs/captain/inbox_pending_conversations_resolution_job_spec.rb +++ b/spec/enterprise/jobs/captain/inbox_pending_conversations_resolution_job_spec.rb @@ -37,6 +37,22 @@ RSpec.describe Captain::InboxPendingConversationsResolutionJob, type: :job do expect(open_conversation.reload.status).to eq('open') end + it 'emits a captain resolved event with the time_based source' do + expect(Captain::ConversationEvents).to receive(:resolved) + .with(conversation: resolvable_pending_conversation, assistant: captain_assistant, source: 'time_based', at: kind_of(Time)) + + described_class.perform_now(inbox) + end + + it 'does not create a captain inference reporting event' do + perform_enqueued_jobs do + described_class.perform_now(inbox) + end + + expect(ReportingEvent.exists?(conversation_id: resolvable_pending_conversation.id, + name: 'conversation_captain_inference_resolved')).to be(false) + end + it 'does not call ConversationCompletionService' do allow(Captain::ConversationCompletionService).to receive(:new) @@ -160,6 +176,13 @@ RSpec.describe Captain::InboxPendingConversationsResolutionJob, type: :job do ) end + it 'emits a captain resolved event with the inference source' do + expect(Captain::ConversationEvents).to receive(:resolved) + .with(conversation: resolvable_pending_conversation, assistant: captain_assistant, source: 'inference', at: kind_of(Time)) + + described_class.perform_now(inbox) + end + it 'creates a captain inference resolved reporting event' do perform_enqueued_jobs do described_class.perform_now(inbox) @@ -259,6 +282,14 @@ RSpec.describe Captain::InboxPendingConversationsResolutionJob, type: :job do ) end + it 'emits a captain handoff event with the inference source' do + expect(Captain::ConversationEvents).to receive(:handed_off) + .with(conversation: resolvable_pending_conversation, assistant: captain_assistant, source: 'inference', + reason_category: :pending_clarification, at: kind_of(Time)) + + described_class.perform_now(inbox) + end + it 'creates a captain inference handoff reporting event' do perform_enqueued_jobs do described_class.perform_now(inbox) diff --git a/spec/enterprise/lib/captain/tools/handoff_tool_spec.rb b/spec/enterprise/lib/captain/tools/handoff_tool_spec.rb index 2047a2da1..d928e7d8f 100644 --- a/spec/enterprise/lib/captain/tools/handoff_tool_spec.rb +++ b/spec/enterprise/lib/captain/tools/handoff_tool_spec.rb @@ -93,6 +93,21 @@ RSpec.describe Captain::Tools::HandoffTool, type: :model do end.not_to change(Message, :count) expect(conversation.reload.status).to eq('pending') end + + it 'emits a captain handoff event with the tool source after the locked handoff completes' do + expect(Captain::ConversationEvents).to receive(:handed_off) + .with(conversation: conversation, assistant: assistant, source: 'tool', at: kind_of(Time)) + + tool.perform(tool_context, reason: 'Customer needs specialized support') + end + + it 'does not emit a captain handoff event when the handoff is skipped as stale' do + create(:message, conversation: conversation, account: account, inbox: inbox, message_type: :incoming) + + expect(Captain::ConversationEvents).not_to receive(:handed_off) + + tool.perform(tool_context, reason: 'Customer needs specialized support') + end end context 'with Captain V1' do @@ -155,6 +170,13 @@ RSpec.describe Captain::Tools::HandoffTool, type: :model do tool.perform(tool_context, reason: 'Test reason') end + it 'emits a captain handoff event with the tool source' do + expect(Captain::ConversationEvents).to receive(:handed_off) + .with(conversation: conversation, assistant: assistant, source: 'tool', at: kind_of(Time)) + + tool.perform(tool_context, reason: 'Test reason') + end + it 'creates a conversation_bot_handoff reporting event' do create(:captain_inbox, captain_assistant: assistant, inbox: inbox) Current.executed_by = assistant diff --git a/spec/enterprise/lib/captain/tools/resolve_conversation_tool_spec.rb b/spec/enterprise/lib/captain/tools/resolve_conversation_tool_spec.rb index 2eae2905a..e3054230c 100644 --- a/spec/enterprise/lib/captain/tools/resolve_conversation_tool_spec.rb +++ b/spec/enterprise/lib/captain/tools/resolve_conversation_tool_spec.rb @@ -29,6 +29,13 @@ RSpec.describe Captain::Tools::ResolveConversationTool do ) end + it 'emits a captain resolution event with the tool source' do + expect(Captain::ConversationEvents).to receive(:resolved) + .with(conversation: conversation, assistant: assistant, source: 'tool', at: kind_of(Time)) + + tool.perform(tool_context, reason: 'Possible spam') + end + it 'creates a conversation_resolved reporting event' do create(:captain_inbox, captain_assistant: assistant, inbox: inbox) @@ -65,6 +72,12 @@ RSpec.describe Captain::Tools::ResolveConversationTool do describe 'resolving an already resolved conversation' do let(:conversation) { create(:conversation, account: account, inbox: inbox, status: :resolved) } + it 'does not emit a captain resolution event' do + expect(Captain::ConversationEvents).not_to receive(:resolved) + + tool.perform(tool_context, reason: 'Possible spam') + end + it 'does not re-resolve and returns an already resolved message' do queue_adapter = ActiveJob::Base.queue_adapter queue_adapter.enqueued_jobs.clear diff --git a/spec/enterprise/listeners/captain/reporting_event_listener_spec.rb b/spec/enterprise/listeners/captain/reporting_event_listener_spec.rb new file mode 100644 index 000000000..8b14a39ef --- /dev/null +++ b/spec/enterprise/listeners/captain/reporting_event_listener_spec.rb @@ -0,0 +1,53 @@ +require 'rails_helper' + +RSpec.describe Captain::ReportingEventListener do + let(:listener) { described_class.instance } + let(:account) { create(:account) } + let(:inbox) { create(:inbox, account: account) } + let(:conversation) { create(:conversation, account: account, inbox: inbox) } + let(:assistant) { create(:captain_assistant, account: account) } + + describe '#captain_conversation_resolved' do + it 'creates a captain inference resolved reporting event for inference resolutions' do + decision_time = conversation.created_at + 60.seconds + event = Events::Base.new( + Events::Types::CAPTAIN_CONVERSATION_RESOLVED, decision_time, + conversation: conversation, assistant: assistant, source: 'inference' + ) + + listener.captain_conversation_resolved(event) + + reporting_event = account.reporting_events.where(name: 'conversation_captain_inference_resolved').first + expect(reporting_event).to be_present + expect(reporting_event.value).to eq 60 + expect(reporting_event.event_end_time).to be_within(1.second).of(decision_time) + end + end + + describe '#captain_conversation_handed_off' do + it 'creates a captain inference handoff reporting event for inference handoffs' do + decision_time = conversation.created_at + 90.seconds + event = Events::Base.new( + Events::Types::CAPTAIN_CONVERSATION_HANDED_OFF, decision_time, + conversation: conversation, assistant: assistant, source: 'inference', reason_category: :pending_clarification + ) + + listener.captain_conversation_handed_off(event) + + reporting_event = account.reporting_events.where(name: 'conversation_captain_inference_handoff').first + expect(reporting_event).to be_present + expect(reporting_event.value).to eq 90 + expect(reporting_event.event_end_time).to be_within(1.second).of(decision_time) + end + + it 'does not create a reporting event for non-inference handoffs' do + event = Events::Base.new( + Events::Types::CAPTAIN_CONVERSATION_HANDED_OFF, Time.zone.now, + conversation: conversation, assistant: assistant, source: 'tool', reason_category: nil + ) + + expect { listener.captain_conversation_handed_off(event) } + .not_to(change { account.reporting_events.count }) + end + end +end diff --git a/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb b/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb index 304a7dd34..494d60dbc 100644 --- a/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb +++ b/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb @@ -259,6 +259,8 @@ RSpec.describe Captain::Assistant::AgentRunnerService do expect(result).to eq({ 'response' => 'conversation_handoff', 'reasoning' => 'Error occurred: Test error', + 'error' => true, + 'error_reason' => 'standard_error', 'handoff_tool_called' => false }) end @@ -287,6 +289,8 @@ RSpec.describe Captain::Assistant::AgentRunnerService do expect(result).to eq({ 'response' => 'conversation_handoff', 'reasoning' => 'Error occurred: Test error', + 'error' => true, + 'error_reason' => 'standard_error', 'handoff_tool_called' => false }) end @@ -311,6 +315,8 @@ RSpec.describe Captain::Assistant::AgentRunnerService do expect(result).to eq({ 'response' => 'conversation_handoff', 'reasoning' => 'Error occurred: Test error', + 'error' => true, + 'error_reason' => 'standard_error', 'handoff_tool_called' => true }) end diff --git a/spec/enterprise/services/captain/conversation_events_spec.rb b/spec/enterprise/services/captain/conversation_events_spec.rb new file mode 100644 index 000000000..b6f22d6cb --- /dev/null +++ b/spec/enterprise/services/captain/conversation_events_spec.rb @@ -0,0 +1,88 @@ +require 'rails_helper' + +RSpec.describe Captain::ConversationEvents do + let(:account) { create(:account) } + let(:conversation) { create(:conversation, account: account) } + let(:assistant) { create(:captain_assistant, account: account) } + let(:timestamp) { Time.zone.now } + + describe '.engaged' do + it 'dispatches the engagement event with normalized context' do + expect(Rails.configuration.dispatcher).to receive(:dispatch).with( + Events::Types::CAPTAIN_CONVERSATION_ENGAGED, + timestamp, + { conversation: conversation, assistant: assistant } + ) + + described_class.engaged(conversation: conversation, assistant: assistant, at: timestamp) + end + end + + describe '.handed_off' do + it 'dispatches the handoff event with source and reason category' do + expect(Rails.configuration.dispatcher).to receive(:dispatch).with( + Events::Types::CAPTAIN_CONVERSATION_HANDED_OFF, + timestamp, + { conversation: conversation, assistant: assistant, source: 'inference', reason_category: :pending_clarification } + ) + + described_class.handed_off( + conversation: conversation, assistant: assistant, source: 'inference', reason_category: :pending_clarification, at: timestamp + ) + end + end + + describe '.resolved' do + it 'dispatches the resolution event with source' do + expect(Rails.configuration.dispatcher).to receive(:dispatch).with( + Events::Types::CAPTAIN_CONVERSATION_RESOLVED, + timestamp, + { conversation: conversation, assistant: assistant, source: 'inference' } + ) + + described_class.resolved(conversation: conversation, assistant: assistant, source: 'inference', at: timestamp) + end + end + + describe '.response_completed' do + it 'dispatches the response completed event with the result message' do + message = create(:message, conversation: conversation, account: account) + + expect(Rails.configuration.dispatcher).to receive(:dispatch).with( + Events::Types::CAPTAIN_RESPONSE_COMPLETED, + timestamp, + { conversation: conversation, assistant: assistant, message: message } + ) + + described_class.response_completed(conversation: conversation, assistant: assistant, message: message, at: timestamp) + end + end + + describe 'when dispatch fails' do + it 'captures the exception instead of raising into the Captain flow' do + conversation + assistant + error = StandardError.new('redis down') + allow(Rails.configuration.dispatcher).to receive(:dispatch).and_raise(error) + expect(ChatwootExceptionTracker).to receive(:new) + .with(error, account: account) + .and_return(instance_double(ChatwootExceptionTracker, capture_exception: true)) + + expect do + described_class.engaged(conversation: conversation, assistant: assistant, at: timestamp) + end.not_to raise_error + end + end + + describe '.response_failed' do + it 'dispatches the response failed event with the failure reason' do + expect(Rails.configuration.dispatcher).to receive(:dispatch).with( + Events::Types::CAPTAIN_RESPONSE_FAILED, + timestamp, + { conversation: conversation, assistant: assistant, reason: 'faraday_bad_request_error' } + ) + + described_class.response_failed(conversation: conversation, assistant: assistant, reason: 'faraday_bad_request_error', at: timestamp) + end + end +end diff --git a/spec/enterprise/services/enterprise/message_templates/hook_execution_service_spec.rb b/spec/enterprise/services/enterprise/message_templates/hook_execution_service_spec.rb index 94894dd6e..b545cd91e 100644 --- a/spec/enterprise/services/enterprise/message_templates/hook_execution_service_spec.rb +++ b/spec/enterprise/services/enterprise/message_templates/hook_execution_service_spec.rb @@ -136,6 +136,21 @@ RSpec.describe MessageTemplates::HookExecutionService do create(:message, conversation: conversation, message_type: :incoming, account: account) end + + it 'emits the engagement event when captain V2 is enabled' do + account.enable_features!('captain_integration_v2') + + expect(Captain::ConversationEvents).to receive(:engaged) + .with(conversation: conversation, assistant: assistant, at: kind_of(Time)) + + create(:message, conversation: conversation, message_type: :incoming, account: account) + end + + it 'does not emit the engagement event when captain V2 is disabled' do + expect(Captain::ConversationEvents).not_to receive(:engaged) + + create(:message, conversation: conversation, message_type: :incoming, account: account) + end end context 'when captain quota is exceeded within business hours' do @@ -157,6 +172,21 @@ RSpec.describe MessageTemplates::HookExecutionService do expect(conversation.reload.status).to eq('open') end + + it 'emits a usage limit handoff event when captain V2 is enabled' do + account.enable_features!('captain_integration_v2') + + expect(Captain::ConversationEvents).to receive(:handed_off) + .with(conversation: conversation, assistant: assistant, source: 'usage_limit', reason_category: :usage_limit, at: kind_of(Time)) + + create(:message, conversation: conversation, message_type: :incoming, account: account) + end + + it 'does not emit a handoff event when captain V2 is disabled' do + expect(Captain::ConversationEvents).not_to receive(:handed_off) + + create(:message, conversation: conversation, message_type: :incoming, account: account) + end end end diff --git a/spec/listeners/reporting_event_listener_spec.rb b/spec/listeners/reporting_event_listener_spec.rb index a497d7dd8..968baa481 100644 --- a/spec/listeners/reporting_event_listener_spec.rb +++ b/spec/listeners/reporting_event_listener_spec.rb @@ -309,38 +309,6 @@ describe ReportingEventListener do end end - describe '#conversation_captain_inference_resolved' do - it 'creates conversation_captain_inference_resolved event' do - expect(account.reporting_events.where(name: 'conversation_captain_inference_resolved').count).to be 0 - decision_time = conversation.created_at + 60.seconds - event = Events::Base.new('conversation.captain_inference_resolved', decision_time, conversation: conversation) - allow(conversation).to receive(:updated_at).and_return(decision_time + 5.minutes) - - listener.conversation_captain_inference_resolved(event) - - reporting_event = account.reporting_events.where(name: 'conversation_captain_inference_resolved').first - expect(reporting_event).to be_present - expect(reporting_event.value).to eq 60 - expect(reporting_event.event_end_time).to be_within(1.second).of(decision_time) - end - end - - describe '#conversation_captain_inference_handoff' do - it 'creates conversation_captain_inference_handoff event' do - expect(account.reporting_events.where(name: 'conversation_captain_inference_handoff').count).to be 0 - decision_time = conversation.created_at + 90.seconds - event = Events::Base.new('conversation.captain_inference_handoff', decision_time, conversation: conversation) - allow(conversation).to receive(:updated_at).and_return(decision_time + 5.minutes) - - listener.conversation_captain_inference_handoff(event) - - reporting_event = account.reporting_events.where(name: 'conversation_captain_inference_handoff').first - expect(reporting_event).to be_present - expect(reporting_event.value).to eq 90 - expect(reporting_event.event_end_time).to be_within(1.second).of(decision_time) - end - end - describe '#conversation_opened' do context 'when conversation is opened for the first time' do let(:new_conversation) { create(:conversation, account: account, inbox: inbox, assignee: user) }