From 56be133bb08f2b6598848e24fe4bc88ac06dab9a Mon Sep 17 00:00:00 2001 From: Sony Mathew Date: Wed, 5 Aug 2026 16:25:30 +0530 Subject: [PATCH] feat(data-imports): add Freshdesk migration (1/3) (#15261) ## Description Adds Freshdesk as an integration import source so administrators can validate a Freshdesk domain and API key, then import contacts, tickets, public replies, customer replies, and private notes while tracking progress from Data Imports. The integration has now been validated against a live Freshdesk trial tenant with contacts, Web Chat and phone tickets, public replies, customer replies, a private note, pagination, requester expansion, and attachment metadata. That validation found and fixed the current Web Chat source mapping and prevented the ticket description from duplicating the initial Web Chat message. Related: #15116 ## Closes Closes [CW-7639](https://linear.app/chatwoot/issue/CW-7639/freshdesk-freshworks-migration) ## Type of change - [x] New feature (non-breaking change which adds functionality) ## What changed - Added a shared source adapter, importer, job, retry, restart, creation, and placeholder inbox contract used by Intercom and Freshdesk. - Added Freshdesk API authentication, contact and ticket pagination, requester expansion, conversation retrieval, normalization, channel grouping, and error handling. - Added current Freshdesk source identifiers through SMS, including Web Chat source `15`, and grouped equivalent sources into placeholder inboxes. - Used Web Chat conversation events as the complete message history so the generated ticket description does not duplicate the initial customer message. - Preserved Freshdesk ticket subjects in source metadata and added a sanitized live-derived Web Chat fixture with structured bodies and attachment metadata. - Added Freshdesk selection, domain and API key validation, and provider-neutral import status handling in the Data Imports UI. ## How to test 1. Enable the data_import feature for an account and open Settings > Data > New import. 2. Select Freshdesk and enter a Freshdesk domain and API key. 3. Select contacts and/or conversations, validate the credentials, and start the import. 4. Confirm progress is displayed and imported tickets appear as resolved conversations in Freshdesk placeholder inboxes with public replies and private notes preserved. 5. Verify Web Chat tickets appear in the Chat placeholder inbox and the initial customer message is imported once. 6. Verify an abandoned import can be restarted and a stalled import can be retried. ## Current scope - **Product decision:** Attachment binaries are intentionally not imported in the current migration scope. Attachment metadata is preserved and messages include a skipped-attachment marker. - Adaptive Retry-After scheduling and handling the 30,000-ticket listing ceiling are covered by stacked follow-up PRs. ## Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [x] I have commented on my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules --- .../v1/accounts/data_imports_controller.rb | 76 +- .../dashboard/i18n/locale/en/settings.json | 22 +- .../routes/dashboard/settings/data/Index.vue | 14 +- .../settings/data/NewImportDialog.vue | 57 +- .../dashboard/settings/data/importSources.js | 12 +- .../dashboard/settings/data/importStatus.js | 11 +- .../data/specs/NewImportDialog.spec.js | 104 ++ .../settings/data/specs/importSources.spec.js | 28 + .../settings/data/specs/importStatus.spec.js | 33 +- app/jobs/data_imports/base_job.rb | 47 + app/jobs/data_imports/contacts_page_job.rb | 26 + .../data_imports/conversations_page_job.rb | 16 + app/jobs/data_imports/freshdesk/base_job.rb | 26 + .../freshdesk/contacts_page_job.rb | 3 + .../freshdesk/conversations_page_job.rb | 3 + app/jobs/data_imports/freshdesk/import_job.rb | 3 + app/jobs/data_imports/import_job.rb | 24 + app/jobs/data_imports/intercom/base_job.rb | 36 +- .../intercom/contacts_page_job.rb | 25 +- .../intercom/conversations_page_job.rb | 15 +- app/jobs/data_imports/intercom/import_job.rb | 23 +- app/models/concerns/out_of_offisable.rb | 17 +- app/models/data_import.rb | 55 +- app/services/data_imports/creation_service.rb | 65 + app/services/data_imports/freshdesk/client.rb | 160 ++ .../freshdesk/credentials_validator.rb | 31 + .../data_imports/freshdesk/importer.rb | 59 + .../freshdesk/message_batch_builder.rb | 10 + .../data_imports/freshdesk/metadata.rb | 51 + .../data_imports/freshdesk/normalizer.rb | 138 ++ .../freshdesk/placeholder_inbox_builder.rb | 5 + app/services/data_imports/freshdesk/source.rb | 196 +++ .../data_imports/freshdesk/source_bucket.rb | 61 + .../data_imports/freshdesk/ticket_page.rb | 62 + app/services/data_imports/importer.rb | 1350 +++++++++++++++++ .../data_imports/intercom/creation_service.rb | 67 +- .../data_imports/intercom/importer.rb | 1323 +--------------- .../intercom/message_batch_builder.rb | 143 +- .../intercom/placeholder_inbox_builder.rb | 40 +- .../data_imports/intercom/restart_service.rb | 55 +- .../data_imports/intercom/retry_service.rb | 28 +- app/services/data_imports/intercom/source.rb | 181 +++ .../data_imports/message_batch_builder.rb | 144 ++ .../data_imports/placeholder_inbox_builder.rb | 65 + app/services/data_imports/restart_service.rb | 56 + app/services/data_imports/retry_service.rb | 28 + app/services/data_imports/source.rb | 21 + .../freshdesk_ticket_limit_error.rb | 13 + spec/factories/data_import.rb | 10 + spec/fixtures/data_import/freshdesk/README.md | 22 + .../data_import/freshdesk/contact.json | 20 + .../freshdesk/ticket_with_conversations.json | 98 ++ .../web_chat_ticket_with_conversations.json | 183 +++ .../freshdesk/import_jobs_spec.rb | 96 ++ spec/models/data_import_spec.rb | 14 + .../api/v1/accounts/data_imports_spec.rb | 123 ++ .../data_imports/freshdesk/client_spec.rb | 90 ++ .../freshdesk/credentials_validator_spec.rb | 39 + .../data_imports/freshdesk/importer_spec.rb | 302 ++++ .../data_imports/freshdesk/normalizer_spec.rb | 106 ++ .../freshdesk/source_bucket_spec.rb | 45 + .../data_imports/freshdesk/source_spec.rb | 128 ++ .../placeholder_inbox_builder_spec.rb | 22 + spec/services/data_imports/source_spec.rb | 26 + 64 files changed, 4538 insertions(+), 1814 deletions(-) create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/specs/NewImportDialog.spec.js create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/specs/importSources.spec.js create mode 100644 app/jobs/data_imports/base_job.rb create mode 100644 app/jobs/data_imports/contacts_page_job.rb create mode 100644 app/jobs/data_imports/conversations_page_job.rb create mode 100644 app/jobs/data_imports/freshdesk/base_job.rb create mode 100644 app/jobs/data_imports/freshdesk/contacts_page_job.rb create mode 100644 app/jobs/data_imports/freshdesk/conversations_page_job.rb create mode 100644 app/jobs/data_imports/freshdesk/import_job.rb create mode 100644 app/jobs/data_imports/import_job.rb create mode 100644 app/services/data_imports/creation_service.rb create mode 100644 app/services/data_imports/freshdesk/client.rb create mode 100644 app/services/data_imports/freshdesk/credentials_validator.rb create mode 100644 app/services/data_imports/freshdesk/importer.rb create mode 100644 app/services/data_imports/freshdesk/message_batch_builder.rb create mode 100644 app/services/data_imports/freshdesk/metadata.rb create mode 100644 app/services/data_imports/freshdesk/normalizer.rb create mode 100644 app/services/data_imports/freshdesk/placeholder_inbox_builder.rb create mode 100644 app/services/data_imports/freshdesk/source.rb create mode 100644 app/services/data_imports/freshdesk/source_bucket.rb create mode 100644 app/services/data_imports/freshdesk/ticket_page.rb create mode 100644 app/services/data_imports/importer.rb create mode 100644 app/services/data_imports/intercom/source.rb create mode 100644 app/services/data_imports/message_batch_builder.rb create mode 100644 app/services/data_imports/placeholder_inbox_builder.rb create mode 100644 app/services/data_imports/restart_service.rb create mode 100644 app/services/data_imports/retry_service.rb create mode 100644 app/services/data_imports/source.rb create mode 100644 lib/custom_exceptions/data_import/freshdesk_ticket_limit_error.rb create mode 100644 spec/fixtures/data_import/freshdesk/README.md create mode 100644 spec/fixtures/data_import/freshdesk/contact.json create mode 100644 spec/fixtures/data_import/freshdesk/ticket_with_conversations.json create mode 100644 spec/fixtures/data_import/freshdesk/web_chat_ticket_with_conversations.json create mode 100644 spec/jobs/data_imports/freshdesk/import_jobs_spec.rb create mode 100644 spec/services/data_imports/freshdesk/client_spec.rb create mode 100644 spec/services/data_imports/freshdesk/credentials_validator_spec.rb create mode 100644 spec/services/data_imports/freshdesk/importer_spec.rb create mode 100644 spec/services/data_imports/freshdesk/normalizer_spec.rb create mode 100644 spec/services/data_imports/freshdesk/source_bucket_spec.rb create mode 100644 spec/services/data_imports/freshdesk/source_spec.rb create mode 100644 spec/services/data_imports/source_spec.rb diff --git a/app/controllers/api/v1/accounts/data_imports_controller.rb b/app/controllers/api/v1/accounts/data_imports_controller.rb index 70432bc33..790cac87d 100644 --- a/app/controllers/api/v1/accounts/data_imports_controller.rb +++ b/app/controllers/api/v1/accounts/data_imports_controller.rb @@ -19,14 +19,12 @@ class Api::V1::Accounts::DataImportsController < Api::V1::Accounts::BaseControll end def validate_source - totals = validate_intercom_source + totals = source_class.credentials_validator(source_params: permitted_params.to_h, import_types: import_types).perform render json: { valid: true, totals: totals } - rescue DataImports::Intercom::Client::AuthenticationError - render_source_validation_error('We could not validate this Intercom access key. Check the key and its permissions.') - rescue DataImports::Intercom::Client::Error - render_source_validation_error('Intercom could not be reached. Please try again.') rescue ArgumentError => e render_source_validation_error(e.message) + rescue StandardError => e + render_source_client_error(e) end def create @@ -36,44 +34,42 @@ class Api::V1::Accounts::DataImportsController < Api::V1::Accounts::BaseControll return end - DataImports::Intercom::ImportJob.perform_later(@data_import, @data_import.active_intercom_import_run_id) + enqueue_import(@data_import) render_show - rescue DataImports::Intercom::Client::AuthenticationError - render_source_validation_error('We could not validate this Intercom access key. Check the key and its permissions.') - rescue DataImports::Intercom::Client::Error - render_source_validation_error('Intercom could not be reached. Please try again.') rescue ArgumentError => e render_source_validation_error(e.message) + rescue StandardError => e + render_source_client_error(e) end def start - restart_service = DataImports::Intercom::RestartService.new(account: Current.account, data_import: @data_import) + restart_service = DataImports::RestartService.new(account: Current.account, data_import: @data_import) restart_result = restart_service.perform @data_import = restart_service.data_import if restart_result == :access_token_missing - render json: { message: 'The Intercom access key for this import is unavailable.' }, status: :unprocessable_entity + render json: { message: "The #{source_name} #{credential_name} for this import is unavailable." }, status: :unprocessable_entity return end - DataImports::Intercom::ImportJob.perform_later(@data_import, @data_import.active_intercom_import_run_id) if restart_result == :enqueue + enqueue_import(@data_import) if restart_result == :enqueue render_show end def retry_import - retry_service = DataImports::Intercom::RetryService.new(account: Current.account, data_import: @data_import) + retry_service = DataImports::RetryService.new(account: Current.account, data_import: @data_import) retry_result = retry_service.perform @data_import = retry_service.data_import case retry_result when :enqueue - DataImports::Intercom::ImportJob.perform_later(@data_import, @data_import.active_intercom_import_run_id) + enqueue_import(@data_import) render_show when :not_stalled - render json: { message: 'This Intercom import is no longer stalled.' }, status: :unprocessable_entity + render json: { message: "This #{source_name} import is no longer stalled." }, status: :unprocessable_entity when :active_import_exists - render json: { message: 'Another Intercom import is already in progress.' }, status: :unprocessable_entity + render json: { message: "Another #{source_name} import is already in progress." }, status: :unprocessable_entity when :access_token_missing - render json: { message: 'The Intercom access key for this import is unavailable.' }, status: :unprocessable_entity + render json: { message: "The #{source_name} #{credential_name} for this import is unavailable." }, status: :unprocessable_entity end end @@ -113,11 +109,11 @@ class Api::V1::Accounts::DataImportsController < Api::V1::Accounts::BaseControll end def permitted_params - params.permit(:name, :source_provider, :access_token, import_types: []) + params.permit(:name, :source_provider, :access_token, :domain, import_types: []) end def creation_service - DataImports::Intercom::CreationService.new( + DataImports::CreationService.new( account: Current.account, initiated_by: Current.user, source_params: permitted_params.to_h @@ -125,24 +121,50 @@ class Api::V1::Accounts::DataImportsController < Api::V1::Accounts::BaseControll end def import_types - return DataImports::Intercom::Importer::DEFAULT_IMPORT_TYPES unless permitted_params.key?(:import_types) + return DataImports::Importer::DEFAULT_IMPORT_TYPES unless permitted_params.key?(:import_types) Array(permitted_params[:import_types]).compact_blank end - def validate_intercom_source - raise ArgumentError, 'Unsupported import source.' unless permitted_params[:source_provider] == 'intercom' + def source_class + provider = @data_import&.source_provider || permitted_params[:source_provider] + DataImports::Source.source_class(provider) + end - DataImports::Intercom::CredentialsValidator.new( - access_token: permitted_params[:access_token], - import_types: import_types - ).perform + def source_name + return 'Integration' unless DataImports::Source.supported?(@data_import&.source_provider || permitted_params[:source_provider]) + + source_class::DISPLAY_NAME + end + + def credential_name + return 'credential' unless DataImports::Source.supported?(@data_import&.source_provider || permitted_params[:source_provider]) + + source_class.credential_name + end + + def enqueue_import(data_import) + DataImports::Source.source_class(data_import.source_provider).import_job_class.perform_later( + data_import, + data_import.active_import_run_id + ) end def render_source_validation_error(message) render json: { valid: false, message: message }, status: :unprocessable_entity end + def render_source_client_error(error) + raise error unless source_class.client_error?(error) + + message = if source_class.authentication_error?(error) + "We could not validate this #{source_name} #{credential_name}. Check the key and its permissions." + else + "#{source_name} could not be reached. Please try again." + end + render_source_validation_error(message) + end + def render_show @import_errors_finder = DataImportErrorFinder.new(@data_import) @skip_logs_finder = DataImportSkipLogFinder.new(@data_import, params) diff --git a/app/javascript/dashboard/i18n/locale/en/settings.json b/app/javascript/dashboard/i18n/locale/en/settings.json index e36732e48..8d1516f3f 100644 --- a/app/javascript/dashboard/i18n/locale/en/settings.json +++ b/app/javascript/dashboard/i18n/locale/en/settings.json @@ -421,6 +421,10 @@ "DESCRIPTION": "Bring your existing contacts and past conversations into this account from another support tool. Each import runs in the background, so you can keep working while it finishes, track its progress, and review anything that was skipped along the way.", "LOADING": "Fetching imports", "DEFAULT_IMPORT_NAME": "Intercom import", + "DEFAULT_IMPORT_NAMES": { + "INTERCOM": "Intercom import", + "FRESHDESK": "Freshdesk import" + }, "TABS": { "IMPORT": "Import", "EXPORT": "Export" @@ -434,9 +438,15 @@ "TITLE": "New import", "SOURCE": "Source", "NAME": "Import name", - "NAME_PLACEHOLDER": "July Intercom migration", + "NAME_PLACEHOLDER": "July support migration", "ACCESS_KEY": "Intercom access key", "ACCESS_KEY_PLACEHOLDER": "Paste your Intercom access key", + "INTERCOM_ACCESS_KEY": "Intercom access key", + "INTERCOM_ACCESS_KEY_PLACEHOLDER": "Paste your Intercom access key", + "FRESHDESK_API_KEY": "Freshdesk API key", + "FRESHDESK_API_KEY_PLACEHOLDER": "Paste your Freshdesk API key", + "FRESHDESK_DOMAIN": "Freshdesk domain", + "FRESHDESK_DOMAIN_PLACEHOLDER": "acme.freshdesk.com", "DATA_TYPES": "Data to import", "VALIDATING": "Validating access key...", "VALID_KEY": "Access key validated.", @@ -510,11 +520,11 @@ } }, "ALERTS": { - "IMPORT_STARTED": "Intercom import has started.", - "IMPORT_RETRIED": "Intercom import has been queued to resume.", - "IMPORT_RETRY_FAILED": "Could not retry the Intercom import.", - "IMPORT_ABANDONED": "Intercom import has been abandoned.", - "IMPORT_FAILED": "Could not start the Intercom import." + "IMPORT_STARTED": "Import has started.", + "IMPORT_RETRIED": "Import has been queued to resume.", + "IMPORT_RETRY_FAILED": "Could not retry the import.", + "IMPORT_ABANDONED": "Import has been abandoned.", + "IMPORT_FAILED": "Could not start the import." } }, "CAPTAIN_SETTINGS": { diff --git a/app/javascript/dashboard/routes/dashboard/settings/data/Index.vue b/app/javascript/dashboard/routes/dashboard/settings/data/Index.vue index 1a11c543a..c34c0a8f6 100644 --- a/app/javascript/dashboard/routes/dashboard/settings/data/Index.vue +++ b/app/javascript/dashboard/routes/dashboard/settings/data/Index.vue @@ -24,7 +24,7 @@ import { formatStatus, importedCount, isActiveImport, - isActiveIntercomImport, + isActiveIntegrationImport, statusDotClass, } from './importStatus'; @@ -53,8 +53,8 @@ const activeTabIndex = computed(() => ); const hasActiveImport = computed(() => dataImports.value.some(isActiveImport)); -const hasActiveIntercomImport = computed(() => - dataImports.value.some(isActiveIntercomImport) +const hasActiveIntegrationImport = computed(() => + dataImports.value.some(isActiveIntegrationImport) ); const dataImportRoute = dataImport => ({ @@ -137,7 +137,7 @@ const openImport = dataImport => { }; const openImportDrawer = () => { - if (!hasActiveIntercomImport.value) showImportDrawer.value = true; + if (!hasActiveIntegrationImport.value) showImportDrawer.value = true; }; const onImportCreated = dataImportId => { @@ -227,9 +227,9 @@ onBeforeUnmount(() => {