From 13de83d1dc98bf26d1c148c91faa89bebc084c32 Mon Sep 17 00:00:00 2001 From: Aakash Bakhle <48802744+aakashb95@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:54:35 +0530 Subject: [PATCH] refactor(captain): route conversation completion by feature (#15317) Conversation completion evaluations now use a dedicated internal LLM feature with GPT 4.1 as the default. The internal route keeps the completion model separate from the installation wide Captain model override and from the assistant route, which can use GPT 5.2 for Captain V2 accounts. Evaluations continue to use the installation API key and do not consume Captain response credits. ## What changed Added an internal `conversation_completion` feature to the LLM model config and excluded internal features from account preferences, the Captain settings API, and Super Admin model overrides. Updated `Captain::ConversationCompletionService` to resolve its model through `Llm::FeatureRouter`. Added focused service and request coverage for model routing and settings visibility. --- config/llm.yml | 4 +++ .../conversation_completion_service.rb | 9 ++++- lib/llm/feature_router.rb | 2 ++ lib/llm/models.rb | 6 +++- .../captain/preferences_controller_spec.rb | 12 +++++-- .../super_admin/accounts_controller_spec.rb | 1 + .../conversation_completion_service_spec.rb | 36 +++++++++++++++++++ 7 files changed, 66 insertions(+), 4 deletions(-) diff --git a/config/llm.yml b/config/llm.yml index 1b702ab6a..e43907234 100644 --- a/config/llm.yml +++ b/config/llm.yml @@ -69,6 +69,10 @@ models: credit_multiplier: 1 features: + conversation_completion: + models: [gpt-4.1] + default: gpt-4.1 + internal: true editor: models: [ diff --git a/enterprise/lib/captain/conversation_completion_service.rb b/enterprise/lib/captain/conversation_completion_service.rb index 37f9add3e..47f035519 100644 --- a/enterprise/lib/captain/conversation_completion_service.rb +++ b/enterprise/lib/captain/conversation_completion_service.rb @@ -16,7 +16,8 @@ class Captain::ConversationCompletionService < Captain::BaseTaskService return default_incomplete_response('No messages found') if content.blank? response = make_api_call( - model: InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value.presence || GPT_MODEL, + feature: 'conversation_completion', + model: self_hosted_model_override, messages: [ { role: 'system', content: prompt_from_file('conversation_completion') }, { role: 'user', content: content } @@ -31,6 +32,12 @@ class Captain::ConversationCompletionService < Captain::BaseTaskService private + def self_hosted_model_override + return unless ChatwootApp.self_hosted_enterprise? + + InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value.presence + end + def prompt_from_file(file_name) Rails.root.join('enterprise/lib/captain/prompts', "#{file_name}.liquid").read end diff --git a/lib/llm/feature_router.rb b/lib/llm/feature_router.rb index d75b06dca..cb5a1f256 100644 --- a/lib/llm/feature_router.rb +++ b/lib/llm/feature_router.rb @@ -24,6 +24,8 @@ module Llm::FeatureRouter private def account_model_override(account, feature_key) + return if Llm::Models.internal_feature?(feature_key) + model = account&.captain_models&.[](feature_key).presence return unless model return model if Llm::Models.valid_model_for?(feature_key, model) diff --git a/lib/llm/models.rb b/lib/llm/models.rb index 896014262..63e8bdfe4 100644 --- a/lib/llm/models.rb +++ b/lib/llm/models.rb @@ -5,7 +5,11 @@ module Llm::Models def providers = CONFIG.fetch('providers') def models = CONFIG.fetch('models') def features = CONFIG.fetch('features') - def feature_keys = features.keys + def feature_keys = features.reject { |_key, config| config['internal'] }.keys + + def internal_feature?(feature) + features.dig(feature.to_s, 'internal') == true + end def feature?(feature) features.key?(feature.to_s) diff --git a/spec/controllers/api/v1/accounts/captain/preferences_controller_spec.rb b/spec/controllers/api/v1/accounts/captain/preferences_controller_spec.rb index 6a28c60da..016d7d4d5 100644 --- a/spec/controllers/api/v1/accounts/captain/preferences_controller_spec.rb +++ b/spec/controllers/api/v1/accounts/captain/preferences_controller_spec.rb @@ -31,6 +31,7 @@ RSpec.describe 'Api::V1::Accounts::Captain::Preferences', type: :request do expect(json_response).to have_key(:providers) expect(json_response).to have_key(:models) expect(json_response).to have_key(:features) + expect(json_response[:features]).not_to have_key(:conversation_completion) end end @@ -150,14 +151,21 @@ RSpec.describe 'Api::V1::Accounts::Captain::Preferences', type: :request do expect(account.reload.captain_models['editor']).to eq('gpt-4.1-mini') end - it 'does not persist unknown captain model feature keys' do + it 'does not persist unknown or internal captain model feature keys' do put "/api/v1/accounts/#{account.id}/captain/preferences", headers: admin.create_new_auth_token, - params: { captain_models: { editor: 'gpt-4.1-mini', unknown_feature: 'gpt-4.1' } }, + params: { + captain_models: { + editor: 'gpt-4.1-mini', + unknown_feature: 'gpt-4.1', + conversation_completion: 'gpt-4.1' + } + }, as: :json expect(response).to have_http_status(:success) expect(account.reload.captain_models).to eq('editor' => 'gpt-4.1-mini') + expect(json_response[:features]).not_to have_key(:conversation_completion) end it 'rejects invalid captain model values for the feature' do diff --git a/spec/controllers/super_admin/accounts_controller_spec.rb b/spec/controllers/super_admin/accounts_controller_spec.rb index 29f93c914..6da2dbc2f 100644 --- a/spec/controllers/super_admin/accounts_controller_spec.rb +++ b/spec/controllers/super_admin/accounts_controller_spec.rb @@ -57,6 +57,7 @@ RSpec.describe 'Super Admin accounts API', type: :request do Llm::Models.feature_keys.each do |feature_key| expect(response.body).to include("account[captain_models][#{feature_key}]") end + expect(response.body).not_to include('account[captain_models][conversation_completion]') document = Nokogiri::HTML(response.body) editor_select = document.at_css('select[name="account[captain_models][editor]"]') diff --git a/spec/enterprise/lib/captain/conversation_completion_service_spec.rb b/spec/enterprise/lib/captain/conversation_completion_service_spec.rb index 525f2954e..b4d787132 100644 --- a/spec/enterprise/lib/captain/conversation_completion_service_spec.rb +++ b/spec/enterprise/lib/captain/conversation_completion_service_spec.rb @@ -19,6 +19,42 @@ RSpec.describe Captain::ConversationCompletionService do end describe '#perform' do + describe 'model routing' do + let(:mock_response) do + instance_double(RubyLLM::Message, content: { 'complete' => true, 'reason' => 'Done' }, input_tokens: 10, output_tokens: 5) + end + + before do + create(:message, conversation: conversation, message_type: :incoming, content: 'Hello') + allow(mock_chat).to receive(:ask).and_return(mock_response) + end + + it 'uses the internal GPT-4.1 route on Chatwoot Cloud' do + allow(ChatwootApp).to receive(:self_hosted_enterprise?).and_return(false) + InstallationConfig.find_or_initialize_by(name: 'CAPTAIN_OPEN_AI_MODEL').update!(value: 'gpt-5.1') + account.enable_features!('captain_integration_v2') + allow(mock_context).to receive(:chat).with(model: 'gpt-4.1').and_return(mock_chat) + + expect(service.perform).to include(complete: true) + end + + it 'uses the installation model on self-hosted Enterprise' do + allow(ChatwootApp).to receive(:self_hosted_enterprise?).and_return(true) + InstallationConfig.find_or_initialize_by(name: 'CAPTAIN_OPEN_AI_MODEL').update!(value: 'gpt-5.1') + allow(mock_context).to receive(:chat).with(model: 'gpt-5.1').and_return(mock_chat) + + expect(service.perform).to include(complete: true) + end + + it 'falls back to the internal GPT-4.1 route when the self-hosted installation model is blank' do + allow(ChatwootApp).to receive(:self_hosted_enterprise?).and_return(true) + InstallationConfig.find_or_initialize_by(name: 'CAPTAIN_OPEN_AI_MODEL').update!(value: '') + allow(mock_context).to receive(:chat).with(model: 'gpt-4.1').and_return(mock_chat) + + expect(service.perform).to include(complete: true) + end + end + context 'when conversation is complete' do let(:mock_response) do instance_double(