feat(voice): transcribe Twilio call recordings (#15241)
Twilio voice calls now get an AI transcript alongside the recording. Once a call ends and its recording is stored, we transcribe it and show the text under the audio player in the call bubble — the same experience WhatsApp voice notes already have. Transcription runs on Captain and consumes Captain response credits, so it only kicks in for accounts with Captain enabled and audio transcriptions turned on. ## How to test 1. On an account with Captain enabled and Settings → Account → Audio transcriptions on, make a call on a Twilio voice inbox and hang up. 2. Open the conversation. The voice call bubble shows the recording player once Twilio delivers the recording. 3. Shortly after, the transcript appears under the player — no refresh needed. 4. Turn audio transcriptions off (or exhaust Captain credits) and repeat: the recording still appears, the transcript does not. ## What changed - `Llm::SpeechToTextService` (new) — blob-in/text-out transcription engine extracted from `Messages::AudioTranscriptionService`: size limit, temp-file download, model resolution via `Llm::FeatureRouter`, the OpenAI call, and Captain credit accounting. `.available_for?` holds the shared gate. - `Messages::AudioTranscriptionService` — now a thin wrapper over that engine; its public contract is unchanged, so `Captain::OpenAiMessageBuilderService` is unaffected. - `Voice::CallTranscriptionService` / `Voice::CallTranscriptionJob` (new) — transcribe `call.recording` into `calls.transcript`, then rebroadcast the message so clients pick it up over the wire. - `Voice::Provider::Twilio::RecordingAttachmentService` — enqueues the job after the recording is attached. The API and frontend needed no changes: `calls.transcript` already existed, `_call.json.jbuilder` already serialized it, and `VoiceCall.vue` already fed it to the audio chip. Nothing had ever written the column. Also wires `instrument_audio_transcription`, which existed but was never called, so both transcription paths now emit LLM spans.
This commit is contained in:
committed by
GitHub
parent
a24f5a3e7a
commit
1e17cbe0e7
@@ -20,42 +20,65 @@ const props = defineProps({
|
||||
});
|
||||
|
||||
const { t } = useI18n();
|
||||
const { highlightContent } = useMessageFormatter();
|
||||
const { getPlainText } = useMessageFormatter();
|
||||
|
||||
const { contentElement, showReadMore, showReadLess, toggleExpanded } =
|
||||
useExpandableContent();
|
||||
|
||||
const messageContent = computed(() => {
|
||||
// We perform search on either content or email subject or transcribed text
|
||||
// Voice-call messages carry a generic content label (e.g. "Twilio voice
|
||||
// call"), so a transcript match would otherwise be hidden behind it.
|
||||
if (
|
||||
props.message.contentType === 'voice_call' &&
|
||||
props.message.call?.transcript
|
||||
) {
|
||||
return props.message.call.transcript;
|
||||
}
|
||||
|
||||
// We perform search on either content, email subject, or transcribed text
|
||||
// (voice-note attachments)
|
||||
if (props.message.content) {
|
||||
return props.message.content;
|
||||
}
|
||||
|
||||
const { content_attributes = {} } = props.message;
|
||||
const { email = {} } = content_attributes || {};
|
||||
const { email = {} } = props.message.contentAttributes || {};
|
||||
if (email.subject) {
|
||||
return email.subject;
|
||||
}
|
||||
|
||||
const audioAttachment = props.message.attachments?.find(
|
||||
attachment => attachment.file_type === 'audio'
|
||||
attachment => attachment.fileType === 'audio'
|
||||
);
|
||||
return audioAttachment?.transcribed_text || '';
|
||||
return audioAttachment?.transcribedText || '';
|
||||
});
|
||||
|
||||
const escapeHtml = html => {
|
||||
const wrapper = document.createElement('p');
|
||||
wrapper.textContent = html;
|
||||
return wrapper.textContent;
|
||||
return wrapper.innerHTML;
|
||||
};
|
||||
|
||||
const highlightedContent = computed(() => {
|
||||
const content = messageContent.value || '';
|
||||
const escapedText = escapeHtml(content);
|
||||
return highlightContent(
|
||||
escapedText,
|
||||
props.searchTerm,
|
||||
'searchkey--highlight'
|
||||
// getPlainText decodes any markdown in the source into literal text (e.g. a
|
||||
// literal "<img ...>" in the transcript stays as visible characters rather
|
||||
// than a tag). escapeHtml then HTML-encodes that text so the highlight
|
||||
// <span> injected below is the only real markup in the final string passed
|
||||
// to v-dompurify-html.
|
||||
const plainText = getPlainText(messageContent.value || '');
|
||||
const escapedText = escapeHtml(plainText);
|
||||
|
||||
const searchTerm = props.searchTerm || '';
|
||||
if (!searchTerm) {
|
||||
return escapedText;
|
||||
}
|
||||
|
||||
const escapedSearchTerm = escapeHtml(searchTerm).replace(
|
||||
/[.*+?^${}()|[\]\\]/g,
|
||||
'\\$&'
|
||||
);
|
||||
return escapedText.replace(
|
||||
new RegExp(`(${escapedSearchTerm})`, 'ig'),
|
||||
'<span class="searchkey--highlight">$1</span>'
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -56,3 +56,5 @@ class Messages::SearchDataPresenter < SimpleDelegator
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
Messages::SearchDataPresenter.prepend_mod_with('Messages::SearchDataPresenter')
|
||||
|
||||
23
enterprise/app/jobs/voice/call_transcription_job.rb
Normal file
23
enterprise/app/jobs/voice/call_transcription_job.rb
Normal file
@@ -0,0 +1,23 @@
|
||||
class Voice::CallTranscriptionJob < ApplicationJob
|
||||
queue_as :low
|
||||
|
||||
# A recording OpenAI rejects (corrupt/unsupported audio) or credentials it refuses
|
||||
# will never succeed on retry — drop the job instead of hammering the API.
|
||||
discard_on Faraday::BadRequestError, Faraday::UnauthorizedError do |job, error|
|
||||
log_context = {
|
||||
call_id: job.arguments.first,
|
||||
job_id: job.job_id,
|
||||
status_code: error.response&.dig(:status)
|
||||
}
|
||||
|
||||
Rails.logger.warn("Discarding call transcription job: #{log_context}")
|
||||
end
|
||||
retry_on ActiveStorage::FileNotFoundError, wait: 2.seconds, attempts: 3
|
||||
|
||||
def perform(call_id)
|
||||
call = Call.find_by(id: call_id)
|
||||
return if call.blank?
|
||||
|
||||
Voice::CallTranscriptionService.new(call: call).perform
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,11 @@
|
||||
module Enterprise::Messages::SearchDataPresenter
|
||||
# Voice-call recordings hang off the Call, not off message attachments, so the
|
||||
# transcript is folded into attachments.transcribed_text — the field search
|
||||
# already queries for audio-message transcriptions.
|
||||
def attachment_data
|
||||
transcript = call&.transcript if content_type == 'voice_call'
|
||||
return super if transcript.blank?
|
||||
|
||||
(super || []) + [{ transcribed_text: transcript }]
|
||||
end
|
||||
end
|
||||
103
enterprise/app/services/llm/speech_to_text_service.rb
Normal file
103
enterprise/app/services/llm/speech_to_text_service.rb
Normal file
@@ -0,0 +1,103 @@
|
||||
# Blob-in, text-out audio transcription shared by voice-note attachments
|
||||
# (Messages::AudioTranscriptionService) and voice-call recordings
|
||||
# (Voice::CallTranscriptionService).
|
||||
class Llm::SpeechToTextService < Llm::LegacyBaseOpenAiService
|
||||
include Integrations::LlmInstrumentation
|
||||
|
||||
# OpenAI's transcription endpoint hard limit is 25 MB *decimal* (25_000_000), not
|
||||
# binary (25.megabytes = 26_214_400) — using the binary form leaks the 25.0–26.2 MB
|
||||
# range to the API as 413s. Long audio (~70+ min Opus) keeps the source audio but
|
||||
# skips transcription.
|
||||
BYTE_LIMIT = 25_000_000
|
||||
|
||||
attr_reader :blob, :account, :transcription_model
|
||||
|
||||
# Transcription runs on Captain's OpenAI credentials and consumes its response credits.
|
||||
def self.available_for?(account)
|
||||
return false unless account.feature_enabled?('captain_integration')
|
||||
return false if account.audio_transcriptions.blank?
|
||||
|
||||
account.usage_limits[:captain][:responses][:current_available].positive?
|
||||
end
|
||||
|
||||
def self.too_large?(blob)
|
||||
blob.present? && blob.byte_size > BYTE_LIMIT
|
||||
end
|
||||
|
||||
def initialize(blob:, account:)
|
||||
super()
|
||||
@blob = blob
|
||||
@account = account
|
||||
@transcription_model = Llm::FeatureRouter.resolve(feature: 'audio_transcription', account: account)[:model]
|
||||
end
|
||||
|
||||
def perform
|
||||
temp_file_path = fetch_audio_file
|
||||
transcribed_text = nil
|
||||
|
||||
File.open(temp_file_path, 'rb') do |file|
|
||||
transcribed_text = instrument_audio_transcription(instrumentation_params(temp_file_path)) do
|
||||
# temperature: 0.0 minimises hallucinations on silence / near-silent
|
||||
# audio; non-zero values trigger spiraling repeats — well-documented
|
||||
# behaviour across OpenAI transcription models.
|
||||
response = @client.audio.transcribe(
|
||||
parameters: {
|
||||
model: transcription_model,
|
||||
file: file,
|
||||
temperature: 0.0
|
||||
}
|
||||
)
|
||||
response['text']
|
||||
end
|
||||
end
|
||||
|
||||
account.increment_response_usage if transcribed_text.present?
|
||||
transcribed_text
|
||||
ensure
|
||||
FileUtils.rm_f(temp_file_path) if temp_file_path.present?
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def fetch_audio_file
|
||||
temp_dir = Rails.root.join('tmp/uploads/audio-transcriptions')
|
||||
FileUtils.mkdir_p(temp_dir)
|
||||
temp_file_name = "#{blob.key}-#{blob.filename}"
|
||||
|
||||
if blob.filename.extension_without_delimiter.blank?
|
||||
extension = extension_from_content_type(blob.content_type)
|
||||
temp_file_name = "#{temp_file_name}.#{extension}" if extension.present?
|
||||
end
|
||||
|
||||
temp_file_path = File.join(temp_dir, temp_file_name)
|
||||
|
||||
File.open(temp_file_path, 'wb') do |file|
|
||||
blob.open do |blob_file|
|
||||
IO.copy_stream(blob_file, file)
|
||||
end
|
||||
end
|
||||
|
||||
temp_file_path
|
||||
end
|
||||
|
||||
def extension_from_content_type(content_type)
|
||||
subtype = content_type.to_s.downcase.split(';').first.to_s.split('/').last.to_s
|
||||
return if subtype.blank?
|
||||
|
||||
{
|
||||
'x-m4a' => 'm4a',
|
||||
'x-wav' => 'wav',
|
||||
'x-mp3' => 'mp3'
|
||||
}.fetch(subtype, subtype)
|
||||
end
|
||||
|
||||
def instrumentation_params(file_path)
|
||||
{
|
||||
span_name: 'llm.messages.audio_transcription',
|
||||
model: transcription_model,
|
||||
account_id: account&.id,
|
||||
feature_name: 'audio_transcription',
|
||||
file_path: file_path
|
||||
}
|
||||
end
|
||||
end
|
||||
@@ -1,26 +1,16 @@
|
||||
class Messages::AudioTranscriptionService< Llm::LegacyBaseOpenAiService
|
||||
include Integrations::LlmInstrumentation
|
||||
|
||||
# OpenAI's transcription endpoint hard limit is 25 MB *decimal* (25_000_000), not
|
||||
# binary (25.megabytes = 26_214_400) — using the binary form leaks the 25.0–26.2 MB
|
||||
# range to the API as 413s. Long audio (~70+ min Opus) keeps the attachment but skips
|
||||
# transcription.
|
||||
TRANSCRIPTION_BYTE_LIMIT = 25_000_000
|
||||
|
||||
attr_reader :attachment, :message, :account, :transcription_model
|
||||
class Messages::AudioTranscriptionService
|
||||
attr_reader :attachment, :message, :account
|
||||
|
||||
def initialize(attachment)
|
||||
super()
|
||||
@attachment = attachment
|
||||
@message = attachment.message
|
||||
@account = message.account
|
||||
@transcription_model = Llm::FeatureRouter.resolve(feature: 'audio_transcription', account: account)[:model]
|
||||
@account = message&.account
|
||||
end
|
||||
|
||||
def perform
|
||||
return { error: 'Transcription limit exceeded' } unless can_transcribe?
|
||||
return { error: 'Message not found' } if message.blank?
|
||||
return { error: 'Audio too large for Whisper' } if audio_too_large?
|
||||
return { error: 'Transcription limit exceeded' } unless Llm::SpeechToTextService.available_for?(account)
|
||||
return { error: 'Audio too large for transcription' } if Llm::SpeechToTextService.too_large?(attachment.file&.blob)
|
||||
|
||||
transcriptions = transcribe_audio
|
||||
Rails.logger.info "Audio transcription successful: #{transcriptions}"
|
||||
@@ -32,77 +22,13 @@ class Messages::AudioTranscriptionService< Llm::LegacyBaseOpenAiService
|
||||
|
||||
private
|
||||
|
||||
def can_transcribe?
|
||||
return false unless account.feature_enabled?('captain_integration')
|
||||
return false if account.audio_transcriptions.blank?
|
||||
|
||||
account.usage_limits[:captain][:responses][:current_available].positive?
|
||||
end
|
||||
|
||||
def audio_too_large?
|
||||
blob = attachment.file&.blob
|
||||
return false unless blob
|
||||
|
||||
blob.byte_size > TRANSCRIPTION_BYTE_LIMIT
|
||||
end
|
||||
|
||||
def fetch_audio_file
|
||||
blob = attachment.file.blob
|
||||
temp_dir = Rails.root.join('tmp/uploads/audio-transcriptions')
|
||||
FileUtils.mkdir_p(temp_dir)
|
||||
temp_file_name = "#{blob.key}-#{blob.filename}"
|
||||
|
||||
if blob.filename.extension_without_delimiter.blank?
|
||||
extension = extension_from_content_type(blob.content_type)
|
||||
temp_file_name = "#{temp_file_name}.#{extension}" if extension.present?
|
||||
end
|
||||
|
||||
temp_file_path = File.join(temp_dir, temp_file_name)
|
||||
|
||||
File.open(temp_file_path, 'wb') do |file|
|
||||
blob.open do |blob_file|
|
||||
IO.copy_stream(blob_file, file)
|
||||
end
|
||||
end
|
||||
|
||||
temp_file_path
|
||||
end
|
||||
|
||||
def transcribe_audio
|
||||
transcribed_text = attachment.meta&.[]('transcribed_text') || ''
|
||||
return transcribed_text if transcribed_text.present?
|
||||
|
||||
temp_file_path = fetch_audio_file
|
||||
transcribed_text = nil
|
||||
|
||||
File.open(temp_file_path, 'rb') do |file|
|
||||
# temperature: 0.0 minimises hallucinations on silence / near-silent
|
||||
# audio; non-zero values trigger spiraling repeats — well-documented
|
||||
# behaviour across OpenAI transcription models.
|
||||
response = @client.audio.transcribe(
|
||||
parameters: {
|
||||
model: transcription_model,
|
||||
file: file,
|
||||
temperature: 0.0
|
||||
}
|
||||
)
|
||||
transcribed_text = response['text']
|
||||
end
|
||||
|
||||
transcribed_text = Llm::SpeechToTextService.new(blob: attachment.file.blob, account: account).perform
|
||||
update_transcription(transcribed_text)
|
||||
transcribed_text
|
||||
ensure
|
||||
FileUtils.rm_f(temp_file_path) if temp_file_path.present?
|
||||
end
|
||||
|
||||
def instrumentation_params(file_path)
|
||||
{
|
||||
span_name: 'llm.messages.audio_transcription',
|
||||
model: transcription_model,
|
||||
account_id: account&.id,
|
||||
feature_name: 'audio_transcription',
|
||||
file_path: file_path
|
||||
}
|
||||
end
|
||||
|
||||
def update_transcription(transcribed_text)
|
||||
@@ -110,21 +36,9 @@ class Messages::AudioTranscriptionService< Llm::LegacyBaseOpenAiService
|
||||
|
||||
attachment.update!(meta: { transcribed_text: transcribed_text })
|
||||
message.reload.send_update_event
|
||||
message.account.increment_response_usage
|
||||
|
||||
return unless ChatwootApp.advanced_search_allowed?
|
||||
|
||||
message.reindex
|
||||
end
|
||||
|
||||
def extension_from_content_type(content_type)
|
||||
subtype = content_type.to_s.downcase.split(';').first.to_s.split('/').last.to_s
|
||||
return if subtype.blank?
|
||||
|
||||
{
|
||||
'x-m4a' => 'm4a',
|
||||
'x-wav' => 'wav',
|
||||
'x-mp3' => 'mp3'
|
||||
}.fetch(subtype, subtype)
|
||||
end
|
||||
end
|
||||
|
||||
37
enterprise/app/services/voice/call_transcription_service.rb
Normal file
37
enterprise/app/services/voice/call_transcription_service.rb
Normal file
@@ -0,0 +1,37 @@
|
||||
class Voice::CallTranscriptionService
|
||||
pattr_initialize [:call!]
|
||||
|
||||
def perform
|
||||
# Split so a publish failure can retry without re-running (and re-charging) transcription.
|
||||
transcribe if call.transcript.blank?
|
||||
publish(call.message) if call.transcript.present?
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def transcribe
|
||||
return unless call.recording.attached?
|
||||
return unless Llm::SpeechToTextService.available_for?(call.account)
|
||||
return if Llm::SpeechToTextService.too_large?(recording_blob)
|
||||
|
||||
transcript = Llm::SpeechToTextService.new(blob: recording_blob, account: call.account).perform
|
||||
call.update!(transcript: transcript) if transcript.present?
|
||||
end
|
||||
|
||||
def publish(message)
|
||||
return if message.blank?
|
||||
|
||||
# Reindex before broadcasting: if reindexing fails and the job retries,
|
||||
# the transcript is already present so only publish reruns. Sending the
|
||||
# update event first would resend it to clients on every such retry.
|
||||
message.reindex if ChatwootApp.advanced_search_allowed?
|
||||
|
||||
# Rebroadcast the message so connected clients pick up the embedded Call
|
||||
# payload (now with transcript) without a refetch.
|
||||
message.reload.send_update_event
|
||||
end
|
||||
|
||||
def recording_blob
|
||||
call.recording.blob
|
||||
end
|
||||
end
|
||||
@@ -19,6 +19,10 @@ class Voice::Provider::Twilio::RecordingAttachmentService
|
||||
# Bump the message updated_at so the message.updated dispatcher rebroadcasts
|
||||
# the embedded Call payload (now with recording_url) to connected clients.
|
||||
call.message&.touch # rubocop:disable Rails/SkipsModelValidations
|
||||
|
||||
# Duplicate callbacks can both clear the outer already_attached? check, so only
|
||||
# the invocation that actually stored the blob pays for transcription.
|
||||
Voice::CallTranscriptionJob.perform_later(call.id) if @persisted
|
||||
end
|
||||
|
||||
private
|
||||
@@ -31,6 +35,7 @@ class Voice::Provider::Twilio::RecordingAttachmentService
|
||||
call.recording_sid = recording_sid
|
||||
call.duration_seconds ||= normalized_recording_duration
|
||||
call.save!
|
||||
@persisted = true
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Messages::SearchDataPresenter do
|
||||
let(:presenter) { described_class.new(message) }
|
||||
let(:account) { create(:account) }
|
||||
let(:conversation) { create(:conversation, account: account) }
|
||||
let(:message) { create(:message, account: account, conversation: conversation, content_type: :voice_call) }
|
||||
|
||||
describe '#search_data' do
|
||||
it 'indexes the call transcript as searchable attachment text' do
|
||||
create(:call, conversation: conversation, message: message, transcript: 'Refund for order 1234')
|
||||
|
||||
expect(presenter.search_data[:attachments]).to eq([{ transcribed_text: 'Refund for order 1234' }])
|
||||
end
|
||||
|
||||
it 'leaves attachments empty when the call has no transcript yet' do
|
||||
create(:call, conversation: conversation, message: message)
|
||||
|
||||
expect(presenter.search_data[:attachments]).to be_nil
|
||||
end
|
||||
|
||||
it 'keeps attachment transcriptions on non voice_call messages' do
|
||||
message.update!(content_type: :text)
|
||||
|
||||
expect(presenter.search_data[:attachments]).to be_nil
|
||||
end
|
||||
end
|
||||
end
|
||||
120
spec/enterprise/services/llm/speech_to_text_service_spec.rb
Normal file
120
spec/enterprise/services/llm/speech_to_text_service_spec.rb
Normal file
@@ -0,0 +1,120 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Llm::SpeechToTextService, type: :service do
|
||||
let(:account) { create(:account, audio_transcriptions: true) }
|
||||
let(:conversation) { create(:conversation, account: account) }
|
||||
let(:message) { create(:message, account: account, conversation: conversation) }
|
||||
let(:attachment) { message.attachments.create!(account: account, file_type: :audio) }
|
||||
let(:service) { described_class.new(blob: attachment.file.blob, account: account) }
|
||||
|
||||
before do
|
||||
InstallationConfig.find_or_create_by!(name: 'CAPTAIN_OPEN_AI_API_KEY') { |config| config.value = 'test-api-key' }
|
||||
InstallationConfig.find_or_create_by!(name: 'CAPTAIN_OPEN_AI_MODEL') { |config| config.value = 'gpt-4o-mini' }
|
||||
|
||||
attachment.file.attach(
|
||||
io: File.open(Rails.public_path.join('audio/widget/ding.mp3')),
|
||||
filename: 'speech',
|
||||
content_type: 'audio/mpeg'
|
||||
)
|
||||
end
|
||||
|
||||
describe '.available_for?' do
|
||||
before do
|
||||
allow(account).to receive(:usage_limits).and_return(
|
||||
{
|
||||
agents: ChatwootApp.max_limit,
|
||||
inboxes: ChatwootApp.max_limit,
|
||||
captain: { responses: { current_available: 100 } }
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
it 'is false when the captain_integration feature is disabled' do
|
||||
account.disable_features!('captain_integration')
|
||||
|
||||
expect(described_class.available_for?(account)).to be(false)
|
||||
end
|
||||
|
||||
it 'is false when audio transcriptions are disabled on the account' do
|
||||
account.enable_features!('captain_integration')
|
||||
account.update!(audio_transcriptions: false)
|
||||
|
||||
expect(described_class.available_for?(account)).to be(false)
|
||||
end
|
||||
|
||||
it 'is false when no captain responses are available' do
|
||||
account.enable_features!('captain_integration')
|
||||
allow(account).to receive(:usage_limits).and_return(captain: { responses: { current_available: 0 } })
|
||||
|
||||
expect(described_class.available_for?(account)).to be(false)
|
||||
end
|
||||
|
||||
it 'is true when the feature, setting and credits are all present' do
|
||||
account.enable_features!('captain_integration')
|
||||
|
||||
expect(described_class.available_for?(account)).to be(true)
|
||||
end
|
||||
end
|
||||
|
||||
describe '.too_large?' do
|
||||
it 'is false when the blob is missing' do
|
||||
expect(described_class.too_large?(nil)).to be(false)
|
||||
end
|
||||
|
||||
it 'is true beyond the byte limit' do
|
||||
allow(attachment.file.blob).to receive(:byte_size).and_return(described_class::BYTE_LIMIT + 1)
|
||||
|
||||
expect(described_class.too_large?(attachment.file.blob)).to be(true)
|
||||
end
|
||||
end
|
||||
|
||||
describe '#fetch_audio_file' do
|
||||
it 'adds extension from content type when filename has no extension' do
|
||||
temp_file_path = service.send(:fetch_audio_file)
|
||||
|
||||
expect(File.extname(temp_file_path)).to eq('.mpeg')
|
||||
ensure
|
||||
FileUtils.rm_f(temp_file_path) if temp_file_path.present?
|
||||
end
|
||||
end
|
||||
|
||||
describe '#perform' do
|
||||
let(:audio_api) { double('audio_api') } # rubocop:disable RSpec/VerifiedDoubles
|
||||
let(:audio_file_path) { Rails.root.join('tmp/speech_to_text_service_spec.mp3').to_s }
|
||||
|
||||
before do
|
||||
File.binwrite(audio_file_path, 'audio')
|
||||
allow(service).to receive(:fetch_audio_file).and_return(audio_file_path)
|
||||
allow(account).to receive(:increment_response_usage)
|
||||
allow(service.client).to receive(:audio).and_return(audio_api)
|
||||
end
|
||||
|
||||
after do
|
||||
FileUtils.rm_f(audio_file_path)
|
||||
end
|
||||
|
||||
it 'uses the audio transcription feature model' do
|
||||
expect(audio_api).to receive(:transcribe).with(
|
||||
parameters: hash_including(model: 'gpt-4o-mini-transcribe', temperature: 0.0)
|
||||
).and_return({ 'text' => 'Audio transcript' })
|
||||
|
||||
expect(service.perform).to eq('Audio transcript')
|
||||
end
|
||||
|
||||
it 'consumes a captain response credit when text comes back' do
|
||||
allow(audio_api).to receive(:transcribe).and_return({ 'text' => 'Audio transcript' })
|
||||
|
||||
service.perform
|
||||
|
||||
expect(account).to have_received(:increment_response_usage)
|
||||
end
|
||||
|
||||
it 'does not consume a credit when the transcription is blank' do
|
||||
allow(audio_api).to receive(:transcribe).and_return({ 'text' => '' })
|
||||
|
||||
service.perform
|
||||
|
||||
expect(account).not_to have_received(:increment_response_usage)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -36,8 +36,7 @@ RSpec.describe Messages::AudioTranscriptionService, type: :service do
|
||||
|
||||
context 'when transcription is successful' do
|
||||
before do
|
||||
# Mock can_transcribe? to return true and transcribe_audio method
|
||||
allow(service).to receive(:can_transcribe?).and_return(true)
|
||||
allow(Llm::SpeechToTextService).to receive(:available_for?).and_return(true)
|
||||
allow(service).to receive(:transcribe_audio).and_return('Hello world transcription')
|
||||
end
|
||||
|
||||
@@ -61,7 +60,7 @@ RSpec.describe Messages::AudioTranscriptionService, type: :service do
|
||||
context 'when attachment already has transcribed text' do
|
||||
before do
|
||||
attachment.update!(meta: { transcribed_text: 'Existing transcription' })
|
||||
allow(service).to receive(:can_transcribe?).and_return(true)
|
||||
allow(Llm::SpeechToTextService).to receive(:available_for?).and_return(true)
|
||||
end
|
||||
|
||||
it 'returns existing transcription without calling API' do
|
||||
@@ -70,66 +69,21 @@ RSpec.describe Messages::AudioTranscriptionService, type: :service do
|
||||
end
|
||||
end
|
||||
|
||||
context 'when the audio exceeds Whisper byte limit' do
|
||||
context 'when the audio exceeds the transcription byte limit' do
|
||||
before do
|
||||
attachment.file.attach(
|
||||
io: File.open(Rails.public_path.join('audio/widget/ding.mp3')),
|
||||
filename: 'large.mp3',
|
||||
content_type: 'audio/mpeg'
|
||||
)
|
||||
allow(service).to receive(:can_transcribe?).and_return(true)
|
||||
allow(attachment.file.blob).to receive(:byte_size).and_return(described_class::TRANSCRIPTION_BYTE_LIMIT + 1)
|
||||
allow(Llm::SpeechToTextService).to receive(:available_for?).and_return(true)
|
||||
allow(attachment.file.blob).to receive(:byte_size).and_return(Llm::SpeechToTextService::BYTE_LIMIT + 1)
|
||||
end
|
||||
|
||||
it 'returns an error without calling Whisper' do
|
||||
it 'returns an error without transcribing' do
|
||||
expect(service).not_to receive(:transcribe_audio)
|
||||
expect(service.perform).to eq({ error: 'Audio too large for Whisper' })
|
||||
expect(service.perform).to eq({ error: 'Audio too large for transcription' })
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#fetch_audio_file' do
|
||||
let(:service) { described_class.new(attachment) }
|
||||
|
||||
before do
|
||||
attachment.file.attach(
|
||||
io: File.open(Rails.public_path.join('audio/widget/ding.mp3')),
|
||||
filename: 'speech',
|
||||
content_type: 'audio/mpeg'
|
||||
)
|
||||
end
|
||||
|
||||
it 'adds extension from content type when filename has no extension' do
|
||||
temp_file_path = service.send(:fetch_audio_file)
|
||||
|
||||
expect(File.extname(temp_file_path)).to eq('.mpeg')
|
||||
ensure
|
||||
FileUtils.rm_f(temp_file_path) if temp_file_path.present?
|
||||
end
|
||||
end
|
||||
|
||||
describe '#transcribe_audio' do
|
||||
let(:service) { described_class.new(attachment) }
|
||||
let(:audio_api) { double('audio_api') } # rubocop:disable RSpec/VerifiedDoubles
|
||||
let(:audio_file_path) { Rails.root.join('tmp/audio_transcription_service_spec.mp3').to_s }
|
||||
|
||||
before do
|
||||
File.binwrite(audio_file_path, 'audio')
|
||||
allow(service).to receive(:fetch_audio_file).and_return(audio_file_path)
|
||||
allow(service).to receive(:update_transcription)
|
||||
allow(service.client).to receive(:audio).and_return(audio_api)
|
||||
end
|
||||
|
||||
after do
|
||||
FileUtils.rm_f(audio_file_path)
|
||||
end
|
||||
|
||||
it 'uses the audio transcription feature model' do
|
||||
expect(audio_api).to receive(:transcribe).with(
|
||||
parameters: hash_including(model: 'gpt-4o-mini-transcribe', temperature: 0.0)
|
||||
).and_return({ 'text' => 'Audio transcript' })
|
||||
|
||||
expect(service.send(:transcribe_audio)).to eq('Audio transcript')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Voice::CallTranscriptionService, type: :service do
|
||||
let(:account) { create(:account, audio_transcriptions: true) }
|
||||
let(:channel) { create(:channel_twilio_sms, :with_voice, account: account, phone_number: '+15551238888') }
|
||||
let(:inbox) { channel.inbox }
|
||||
let(:conversation) { create(:conversation, account: account, inbox: inbox) }
|
||||
let(:message) { create(:message, account: account, inbox: inbox, conversation: conversation, content_type: :voice_call) }
|
||||
let(:call) do
|
||||
create(:call, account: account, inbox: inbox, conversation: conversation, contact: conversation.contact, status: 'completed', message: message)
|
||||
end
|
||||
|
||||
before do
|
||||
allow(Llm::SpeechToTextService).to receive(:available_for?).and_return(true)
|
||||
call.recording.attach(
|
||||
io: File.open(Rails.public_path.join('audio/widget/ding.mp3')),
|
||||
filename: 'call-recording.mp3',
|
||||
content_type: 'audio/mpeg'
|
||||
)
|
||||
end
|
||||
|
||||
describe '#perform' do
|
||||
it 'stores the transcript on the call' do
|
||||
allow(Llm::SpeechToTextService).to receive(:new).and_return(
|
||||
instance_double(Llm::SpeechToTextService, perform: 'Hello, how can I help?')
|
||||
)
|
||||
|
||||
described_class.new(call: call).perform
|
||||
|
||||
expect(call.reload.transcript).to eq('Hello, how can I help?')
|
||||
end
|
||||
|
||||
it 'skips calls that are already transcribed' do
|
||||
call.update!(transcript: 'Existing transcript')
|
||||
|
||||
expect(Llm::SpeechToTextService).not_to receive(:new)
|
||||
|
||||
described_class.new(call: call).perform
|
||||
end
|
||||
|
||||
it 'skips calls without a recording' do
|
||||
call.recording.purge
|
||||
|
||||
expect(Llm::SpeechToTextService).not_to receive(:new)
|
||||
|
||||
described_class.new(call: call).perform
|
||||
end
|
||||
|
||||
it 'skips when transcription is unavailable for the account' do
|
||||
allow(Llm::SpeechToTextService).to receive(:available_for?).and_return(false)
|
||||
|
||||
expect(Llm::SpeechToTextService).not_to receive(:new)
|
||||
|
||||
described_class.new(call: call).perform
|
||||
end
|
||||
|
||||
it 'leaves the transcript blank when nothing comes back' do
|
||||
allow(Llm::SpeechToTextService).to receive(:new).and_return(
|
||||
instance_double(Llm::SpeechToTextService, perform: '')
|
||||
)
|
||||
|
||||
described_class.new(call: call).perform
|
||||
|
||||
expect(call.reload.transcript).to be_nil
|
||||
end
|
||||
|
||||
it 'reindexes before broadcasting so a retry after a reindex failure does not resend the update event' do
|
||||
call.update!(transcript: 'Existing transcript')
|
||||
allow(ChatwootApp).to receive(:advanced_search_allowed?).and_return(true)
|
||||
allow(message).to receive(:reindex).and_raise(StandardError, 'reindex boom')
|
||||
|
||||
expect(message).not_to receive(:send_update_event)
|
||||
expect { described_class.new(call: call).perform }.to raise_error(StandardError, 'reindex boom')
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -111,6 +111,28 @@ RSpec.describe Voice::Provider::Twilio::RecordingAttachmentService do
|
||||
expect(call.reload.recording.blob.checksum).to be_present
|
||||
end
|
||||
|
||||
it 'enqueues transcription for the invocation that stored the recording' do
|
||||
expect { perform_service }.to have_enqueued_job(Voice::CallTranscriptionJob).with(call.id)
|
||||
end
|
||||
|
||||
it 'does not enqueue transcription when another invocation already stored the recording' do
|
||||
perform_service
|
||||
|
||||
expect { perform_service }.not_to have_enqueued_job(Voice::CallTranscriptionJob)
|
||||
end
|
||||
|
||||
it 'does not enqueue transcription when it loses the race inside the lock' do
|
||||
call.recording.attach(io: StringIO.new('AUDIO'), filename: 'winner.wav', content_type: 'audio/wav')
|
||||
# The outer guard passes while recording_sid is still blank; the winning writer's
|
||||
# value only lands once this invocation takes the lock, so the inner guard trips.
|
||||
allow(call).to receive(:with_lock) do |&block|
|
||||
call.recording_sid = recording_sid
|
||||
block.call
|
||||
end
|
||||
|
||||
expect { perform_service }.not_to have_enqueued_job(Voice::CallTranscriptionJob)
|
||||
end
|
||||
|
||||
it 'is a no-op when recording_sid is blank' do
|
||||
expect { perform_service(recording_sid: '') }.not_to change { call.reload.recording.attached? }.from(false)
|
||||
expect(SafeFetch).not_to have_received(:fetch)
|
||||
|
||||
Reference in New Issue
Block a user