Recovered state: integrated TrendSurferAgent, restored frontend/backend files, and cleaned up recovery scripts

backend/api/content_planning/services/ai_analytics_service.py

@@ -13,7 +13,8 @@ import time
 from services.content_planning_db import ContentPlanningDBService
 from services.ai_analysis_db_service import AIAnalysisDBService
 from services.ai_analytics_service import AIAnalyticsService
-from services.onboarding.data_service import OnboardingDataService
+from services.database import SessionLocal
+from api.content_planning.services.content_strategy.onboarding import OnboardingDataIntegrationService
 
 # Import utilities
 from ..utils.error_handlers import ContentPlanningErrorHandler
@@ -26,15 +27,16 @@ class ContentPlanningAIAnalyticsService:
     def __init__(self):
         self.ai_analysis_db_service = AIAnalysisDBService()
         self.ai_analytics_service = AIAnalyticsService()
-        self.onboarding_service = OnboardingDataService()
+        self.onboarding_integration_service = OnboardingDataIntegrationService()
     
-    async def analyze_content_evolution(self, strategy_id: int, time_period: str = "30d") -> Dict[str, Any]:
+    async def analyze_content_evolution(self, user_id: int, strategy_id: int, time_period: str = "30d") -> Dict[str, Any]:
         """Analyze content evolution over time for a specific strategy."""
         try:
-            logger.info(f"Starting content evolution analysis for strategy {strategy_id}")
+            logger.info(f"Starting content evolution analysis for strategy {strategy_id} (user {user_id})")
             
             # Perform content evolution analysis
             evolution_analysis = await self.ai_analytics_service.analyze_content_evolution(
+                user_id=user_id,
                 strategy_id=strategy_id,
                 time_period=time_period
             )
@@ -55,13 +57,14 @@ class ContentPlanningAIAnalyticsService:
             logger.error(f"Error analyzing content evolution: {str(e)}")
             raise ContentPlanningErrorHandler.handle_general_error(e, "analyze_content_evolution")
     
-    async def analyze_performance_trends(self, strategy_id: int, metrics: Optional[List[str]] = None) -> Dict[str, Any]:
+    async def analyze_performance_trends(self, user_id: int, strategy_id: int, metrics: Optional[List[str]] = None) -> Dict[str, Any]:
         """Analyze performance trends for content strategy."""
         try:
-            logger.info(f"Starting performance trends analysis for strategy {strategy_id}")
+            logger.info(f"Starting performance trends analysis for strategy {strategy_id} (user {user_id})")
             
             # Perform performance trends analysis
             trends_analysis = await self.ai_analytics_service.analyze_performance_trends(
+                user_id=user_id,
                 strategy_id=strategy_id,
                 metrics=metrics
             )
@@ -191,24 +194,31 @@ class ContentPlanningAIAnalyticsService:
             # 🚨 CRITICAL: Always run fresh AI analysis for refresh operations
             logger.info(f"🔄 Running FRESH AI analysis for user {current_user_id} (force_refresh: {force_refresh})")
             
-            # Get personalized inputs from onboarding data
-            personalized_inputs = self.onboarding_service.get_personalized_ai_inputs(current_user_id)
+            # Get personalized inputs from onboarding data (SSOT)
+            db = SessionLocal()
+            try:
+                personalized_inputs = await self.onboarding_integration_service.process_onboarding_data(str(current_user_id), db)
+            finally:
+                db.close()
             
             logger.info(f"📊 Using personalized inputs: {len(personalized_inputs)} data points")
             
             # Generate real AI insights using personalized data
             logger.info("🔍 Generating performance analysis...")
             performance_analysis = await self.ai_analytics_service.analyze_performance_trends(
+                user_id=current_user_id,
                 strategy_id=strategy_id or 1
             )
             
             logger.info("🧠 Generating strategic intelligence...")
             strategic_intelligence = await self.ai_analytics_service.generate_strategic_intelligence(
+                user_id=current_user_id,
                 strategy_id=strategy_id or 1
             )
             
             logger.info("📈 Analyzing content evolution...")
             evolution_analysis = await self.ai_analytics_service.analyze_content_evolution(
+                user_id=current_user_id,
                 strategy_id=strategy_id or 1
             )
             
@@ -255,9 +265,9 @@ class ContentPlanningAIAnalyticsService:
                 "data_source": "ai_analysis",
                 "user_profile": {
                     "website_url": personalized_inputs.get('website_analysis', {}).get('website_url', ''),
-                    "content_types": personalized_inputs.get('website_analysis', {}).get('content_types', []),
-                    "target_audience": personalized_inputs.get('website_analysis', {}).get('target_audience', []),
-                    "industry_focus": personalized_inputs.get('website_analysis', {}).get('industry_focus', 'general')
+                    "content_types": personalized_inputs.get('canonical_profile', {}).get('content_types', []),
+                    "target_audience": personalized_inputs.get('canonical_profile', {}).get('target_audience', []),
+                    "industry_focus": personalized_inputs.get('canonical_profile', {}).get('industry', 'general')
                 }
             }
             

backend/api/content_planning/services/content_strategy/ai_generation/strategy_generator.py

@@ -75,27 +75,27 @@ class AIStrategyGenerator:
             base_strategy = await self._generate_base_strategy_fields(user_id, context)
             
             # Step 2: Generate strategic insights and recommendations
-            strategic_insights = await self._generate_strategic_insights(base_strategy, context)
+            strategic_insights = await self._generate_strategic_insights(base_strategy, context, user_id=user_id)
             if strategic_insights.get("ai_generation_failed"):
                 failed_components.append("strategic_insights")
             
             # Step 3: Generate competitive analysis
-            competitive_analysis = await self._generate_competitive_analysis(base_strategy, context)
+            competitive_analysis = await self._generate_competitive_analysis(base_strategy, context, user_id=user_id)
             if competitive_analysis.get("ai_generation_failed"):
                 failed_components.append("competitive_analysis")
             
             # Step 4: Generate performance predictions
-            performance_predictions = await self._generate_performance_predictions(base_strategy, context)
+            performance_predictions = await self._generate_performance_predictions(base_strategy, context, user_id=user_id)
             if performance_predictions.get("ai_generation_failed"):
                 failed_components.append("performance_predictions")
             
             # Step 5: Generate implementation roadmap
-            implementation_roadmap = await self._generate_implementation_roadmap(base_strategy, context)
+            implementation_roadmap = await self._generate_implementation_roadmap(base_strategy, context, user_id=user_id)
             if implementation_roadmap.get("ai_generation_failed"):
                 failed_components.append("implementation_roadmap")
             
             # Step 6: Generate risk assessment
-            risk_assessment = await self._generate_risk_assessment(base_strategy, context)
+            risk_assessment = await self._generate_risk_assessment(base_strategy, context, user_id=user_id)
             if risk_assessment.get("ai_generation_failed"):
                 failed_components.append("risk_assessment")
             
@@ -169,7 +169,7 @@ class AIStrategyGenerator:
             self.logger.error(f"Error generating base strategy fields: {str(e)}")
             raise
 
-    async def _generate_strategic_insights(self, base_strategy: Dict[str, Any], context: Dict[str, Any], ai_manager: Optional[Any] = None) -> Dict[str, Any]:
+    async def _generate_strategic_insights(self, base_strategy: Dict[str, Any], context: Dict[str, Any], user_id: Optional[int] = None, ai_manager: Optional[Any] = None) -> Dict[str, Any]:
         """Generate strategic insights using AI."""
         try:
             logger.info("🧠 Generating strategic insights...")
@@ -222,7 +222,8 @@ class AIStrategyGenerator:
             response = await ai_manager.execute_structured_json_call(
                 AIServiceType.STRATEGIC_INTELLIGENCE, 
                 prompt, 
-                schema
+                schema,
+                user_id=str(user_id) if user_id else None
             )
             
             if not response or not response.get("data"):
@@ -306,7 +307,8 @@ class AIStrategyGenerator:
             response = await ai_manager.execute_structured_json_call(
                 AIServiceType.MARKET_POSITION_ANALYSIS, 
                 prompt, 
-                schema
+                schema,
+                user_id=str(user_id) if user_id else None
             )
             
             if not response or not response.get("data"):
@@ -339,7 +341,7 @@ class AIStrategyGenerator:
                 "failure_reason": str(e)
             }
 
-    async def _generate_content_calendar(self, base_strategy: Dict[str, Any], context: Dict[str, Any], ai_manager: Optional[Any] = None) -> Dict[str, Any]:
+    async def _generate_content_calendar(self, base_strategy: Dict[str, Any], context: Dict[str, Any], user_id: Optional[int] = None, ai_manager: Optional[Any] = None) -> Dict[str, Any]:
         """Generate content calendar using AI."""
         try:
             logger.info("📅 Generating content calendar...")
@@ -442,7 +444,8 @@ class AIStrategyGenerator:
             response = await ai_manager.execute_structured_json_call(
                 AIServiceType.CONTENT_SCHEDULE_GENERATION, 
                 prompt, 
-                schema
+                schema,
+                user_id=str(user_id) if user_id else None
             )
             
             if not response or not response.get("data"):
@@ -455,7 +458,7 @@ class AIStrategyGenerator:
             logger.error(f"❌ Error generating content calendar: {str(e)}")
             raise RuntimeError(f"Failed to generate content calendar: {str(e)}")
 
-    async def _generate_performance_predictions(self, base_strategy: Dict[str, Any], context: Dict[str, Any], ai_manager: Optional[Any] = None) -> Dict[str, Any]:
+    async def _generate_performance_predictions(self, base_strategy: Dict[str, Any], context: Dict[str, Any], user_id: Optional[int] = None, ai_manager: Optional[Any] = None) -> Dict[str, Any]:
         """Generate performance predictions using AI."""
         try:
             logger.info("📊 Generating performance predictions...")
@@ -525,7 +528,8 @@ class AIStrategyGenerator:
             response = await ai_manager.execute_structured_json_call(
                 AIServiceType.PERFORMANCE_PREDICTION, 
                 prompt, 
-                schema
+                schema,
+                user_id=str(user_id) if user_id else None
             )
             
             if not response or not response.get("data"):
@@ -551,7 +555,7 @@ class AIStrategyGenerator:
                 "failure_reason": str(e)
             }
 
-    async def _generate_implementation_roadmap(self, base_strategy: Dict[str, Any], context: Dict[str, Any], ai_manager: Optional[Any] = None) -> Dict[str, Any]:
+    async def _generate_implementation_roadmap(self, base_strategy: Dict[str, Any], context: Dict[str, Any], user_id: Optional[int] = None, ai_manager: Optional[Any] = None) -> Dict[str, Any]:
         """Generate implementation roadmap using AI."""
         try:
             logger.info("🗺️ Generating implementation roadmap...")

backend/api/content_planning/services/content_strategy/core/strategy_service.py

@@ -10,7 +10,6 @@ from sqlalchemy.orm import Session
 
 # Import database models
 from models.enhanced_strategy_models import EnhancedContentStrategy, EnhancedAIAnalysisResult, OnboardingDataIntegration
-from models.onboarding import OnboardingSession, WebsiteAnalysis, ResearchPreferences, APIKey
 
 # Import modular services
 from ..ai_analysis.ai_recommendations import AIRecommendationsService
@@ -177,7 +176,7 @@ class EnhancedStrategyService:
             db.rollback()
             raise
 
-    async def get_enhanced_strategies(self, user_id: Optional[int] = None, strategy_id: Optional[int] = None, db: Session = None) -> Dict[str, Any]:
+    async def get_enhanced_strategies(self, user_id: Optional[str] = None, strategy_id: Optional[int] = None, db: Session = None) -> Dict[str, Any]:
         """Get enhanced content strategies with comprehensive data and AI recommendations."""
         try:
             logger.info(f"🚀 Starting enhanced strategy analysis for user: {user_id}, strategy: {strategy_id}")
@@ -261,102 +260,115 @@ class EnhancedStrategyService:
             logger.error(f"❌ Error retrieving enhanced strategies: {str(e)}")
             raise
 
-    async def _enhance_strategy_with_onboarding_data(self, strategy: EnhancedContentStrategy, user_id: int, db: Session) -> None:
-        """Enhance strategy with intelligent auto-population from onboarding data."""
+    async def _enhance_strategy_with_onboarding_data(self, strategy: EnhancedContentStrategy, user_id: str, db: Session) -> None:
+        """Enhance strategy with intelligent auto-population from canonical onboarding data."""
         try:
             logger.info(f"Enhancing strategy with onboarding data for user: {user_id}")
-            
-            # Get onboarding session
-            onboarding_session = db.query(OnboardingSession).filter(
-                OnboardingSession.user_id == user_id
-            ).first()
-            
-            if not onboarding_session:
-                logger.info("No onboarding session found for user")
-                return
-            
-            # Get website analysis data
-            website_analysis = db.query(WebsiteAnalysis).filter(
-                WebsiteAnalysis.session_id == onboarding_session.id
-            ).first()
-            
-            # Get research preferences data
-            research_preferences = db.query(ResearchPreferences).filter(
-                ResearchPreferences.session_id == onboarding_session.id
-            ).first()
-            
-            # Get API keys data
-            api_keys = db.query(APIKey).filter(
-                APIKey.session_id == onboarding_session.id
-            ).all()
-            
-            # Auto-populate fields from onboarding data
+
+            integrated_data = await self.onboarding_data_service.process_onboarding_data(user_id, db)
+            canonical_profile = integrated_data.get('canonical_profile') or {}
+
+            website_analysis = integrated_data.get('website_analysis') or {}
+            research_preferences = integrated_data.get('research_preferences') or {}
+            competitor_analysis = integrated_data.get('competitor_analysis') or []
+            api_keys_data = integrated_data.get('api_keys_data') or {}
+
             auto_populated_fields = {}
             data_sources = {}
-            
-            if website_analysis:
-                # Extract content preferences from writing style
-                if website_analysis.writing_style:
-                    strategy.content_preferences = extract_content_preferences_from_style(
-                        website_analysis.writing_style
-                    )
+
+            # Prioritize Canonical Profile for merged insights
+            if canonical_profile:
+                if canonical_profile.get('target_audience'):
+                    strategy.target_audience = canonical_profile.get('target_audience')
+                    auto_populated_fields['target_audience'] = 'canonical_profile'
+                
+                if canonical_profile.get('industry'):
+                    strategy.industry = canonical_profile.get('industry')
+                    auto_populated_fields['industry'] = 'canonical_profile'
+
+                if canonical_profile.get('content_types'):
+                    strategy.preferred_formats = canonical_profile.get('content_types')
+                    auto_populated_fields['preferred_formats'] = 'canonical_profile'
+
+            if isinstance(website_analysis, dict) and website_analysis:
+                writing_style = website_analysis.get('writing_style') or {}
+                if isinstance(writing_style, dict) and writing_style:
+                    strategy.content_preferences = extract_content_preferences_from_style(writing_style)
                     auto_populated_fields['content_preferences'] = 'website_analysis'
-                
-                # Extract target audience from analysis
-                if website_analysis.target_audience:
-                    strategy.target_audience = website_analysis.target_audience
-                    auto_populated_fields['target_audience'] = 'website_analysis'
-                
-                # Extract brand voice from style guidelines
-                if website_analysis.style_guidelines:
-                    strategy.brand_voice = extract_brand_voice_from_guidelines(
-                        website_analysis.style_guidelines
-                    )
+
+                # Fallback to website_analysis if not in canonical_profile
+                if 'target_audience' not in auto_populated_fields:
+                    target_audience = website_analysis.get('target_audience')
+                    if target_audience:
+                        strategy.target_audience = target_audience
+                        auto_populated_fields['target_audience'] = 'website_analysis'
+
+                style_guidelines = website_analysis.get('style_guidelines') or {}
+                if isinstance(style_guidelines, dict) and style_guidelines:
+                    strategy.brand_voice = extract_brand_voice_from_guidelines(style_guidelines)
                     auto_populated_fields['brand_voice'] = 'website_analysis'
-                
-                data_sources['website_analysis'] = website_analysis.to_dict()
-            
-            if research_preferences:
-                # Extract content types from research preferences
-                if research_preferences.content_types:
-                    strategy.preferred_formats = research_preferences.content_types
-                    auto_populated_fields['preferred_formats'] = 'research_preferences'
-                
-                # Extract writing style from preferences
-                if research_preferences.writing_style:
-                    strategy.editorial_guidelines = extract_editorial_guidelines_from_style(
-                        research_preferences.writing_style
-                    )
+
+                data_sources['website_analysis'] = website_analysis
+
+            if isinstance(research_preferences, dict) and research_preferences:
+                # Fallback to research_preferences if not in canonical_profile
+                if 'preferred_formats' not in auto_populated_fields:
+                    content_types = research_preferences.get('content_types')
+                    if content_types:
+                        strategy.preferred_formats = content_types
+                        auto_populated_fields['preferred_formats'] = 'research_preferences'
+
+                prefs_writing_style = research_preferences.get('writing_style') or {}
+                if isinstance(prefs_writing_style, dict) and prefs_writing_style:
+                    strategy.editorial_guidelines = extract_editorial_guidelines_from_style(prefs_writing_style)
                     auto_populated_fields['editorial_guidelines'] = 'research_preferences'
+
+                data_sources['research_preferences'] = research_preferences
+
+            # Integrate Competitor Analysis (Step 3)
+            if competitor_analysis:
+                competitors = []
+                for comp in competitor_analysis:
+                    # Prefer domain, then title, then url
+                    # Handle both dict and object (though integrated_data usually returns dicts via to_dict)
+                    if isinstance(comp, dict):
+                        name = comp.get('competitor_domain') or comp.get('title') or comp.get('competitor_url')
+                    else:
+                        name = getattr(comp, 'competitor_domain', None) or getattr(comp, 'competitor_url', None)
+                    
+                    if name:
+                        competitors.append(name)
                 
-                data_sources['research_preferences'] = research_preferences.to_dict()
-            
-            # Create onboarding data integration record
+                if competitors:
+                    # Limit to top 10 to avoid overwhelming the strategy
+                    strategy.top_competitors = competitors[:10]
+                    auto_populated_fields['top_competitors'] = 'competitor_analysis'
+                    data_sources['competitor_analysis'] = competitor_analysis
+
             integration = OnboardingDataIntegration(
                 user_id=user_id,
                 strategy_id=strategy.id,
                 website_analysis_data=data_sources.get('website_analysis'),
                 research_preferences_data=data_sources.get('research_preferences'),
-                api_keys_data=[key.to_dict() for key in api_keys] if api_keys else None,
+                api_keys_data=api_keys_data,
                 auto_populated_fields=auto_populated_fields,
                 field_mappings=create_field_mappings(),
                 data_quality_scores=calculate_data_quality_scores(data_sources),
-                confidence_levels={},  # Will be calculated by data quality service
-                data_freshness={}  # Will be calculated by data quality service
+                confidence_levels={},
+                data_freshness={}
             )
-            
+
             db.add(integration)
             db.commit()
-            
-            # Update strategy with onboarding data used
+
             strategy.onboarding_data_used = {
                 'auto_populated_fields': auto_populated_fields,
                 'data_sources': list(data_sources.keys()),
                 'integration_id': integration.id
             }
-            
+
             logger.info(f"Strategy enhanced with onboarding data: {len(auto_populated_fields)} fields auto-populated")
-            
+
         except Exception as e:
             logger.error(f"Error enhancing strategy with onboarding data: {str(e)}")
             # Don't raise error, just log it as this is enhancement, not core functionality
@@ -581,4 +593,4 @@ class EnhancedStrategyService:
     def _convert_to_xml(self, data: Dict[str, Any]) -> str:
         """Convert data to XML format (placeholder implementation)."""
         # This would be implemented with proper XML conversion
-        return f"<strategy>{str(data)}</strategy>" 
+        return f"<strategy>{str(data)}</strategy>" 

backend/api/content_planning/services/content_strategy/onboarding/data_integration.py

@@ -3,7 +3,7 @@ Onboarding Data Integration Service
 Onboarding data integration and processing.
 """
 
-import logging
+from utils.logger_utils import get_service_logger
 from typing import Dict, Any, Optional, List
 from datetime import datetime, timedelta
 from sqlalchemy.orm import Session
@@ -19,11 +19,16 @@ from models.onboarding import (
     ResearchPreferences,
     APIKey,
     PersonaData,
-    CompetitorAnalysis
+    CompetitorAnalysis,
+    SEOPageAudit
+)
+from models.website_analysis_monitoring_models import (
+    DeepCompetitorAnalysisTask,
+    DeepCompetitorAnalysisExecutionLog
 )
 import os
 
-logger = logging.getLogger(__name__)
+logger = get_service_logger("onboarding.data_integration")
 
 class OnboardingDataIntegrationService:
     """Service for onboarding data integration and processing."""
@@ -32,6 +37,162 @@ class OnboardingDataIntegrationService:
         self.data_freshness_threshold = timedelta(hours=24)
         self.max_analysis_age = timedelta(days=7)
 
+    def get_integrated_data_sync(self, user_id: str, db: Session) -> Dict[str, Any]:
+        """Synchronous version of process_onboarding_data for sync contexts.
+           Note: Does not include async data sources like GSC/Bing analytics.
+        """
+        try:
+            # Get all onboarding data sources (DB only)
+            website_analysis = self._get_website_analysis(user_id, db)
+            research_preferences = self._get_research_preferences(user_id, db)
+            api_keys_data = self._get_api_keys_data(user_id, db)
+            onboarding_session = self._get_onboarding_session(user_id, db)
+            persona_data = self._get_persona_data(user_id, db)
+            competitor_analysis = self._get_competitor_analysis(user_id, db)
+            deep_competitor_analysis = self._get_deep_competitor_analysis(user_id, db)
+            
+            # Skip async sources
+            gsc_analytics = {}
+            bing_analytics = {}
+
+            canonical_profile = self._build_canonical_profile(
+                website_analysis,
+                research_preferences,
+                persona_data,
+                onboarding_session,
+                competitor_analysis,
+                deep_competitor_analysis
+            )
+
+            integrated_data = {
+                'website_analysis': website_analysis,
+                'research_preferences': research_preferences,
+                'api_keys_data': api_keys_data,
+                'onboarding_session': onboarding_session,
+                'persona_data': persona_data,
+                'competitor_analysis': competitor_analysis,
+                'deep_competitor_analysis': deep_competitor_analysis,
+                'gsc_analytics': gsc_analytics,
+                'bing_analytics': bing_analytics,
+                'canonical_profile': canonical_profile,
+                'data_quality': self._assess_data_quality(website_analysis, research_preferences, api_keys_data, persona_data, competitor_analysis, gsc_analytics, bing_analytics),
+                'processing_timestamp': datetime.utcnow().isoformat()
+            }
+            
+            return integrated_data
+            
+        except Exception as e:
+            logger.error(f"Error processing onboarding data (sync) for user {user_id}: {str(e)}")
+            return self._get_fallback_data()
+
+    async def refresh_integrated_data(self, user_id: str, db: Session) -> None:
+        """
+        Refresh and store integrated data (DB-only sources) to ensure SSOT is up-to-date.
+        This is a lightweight version of process_onboarding_data suitable for calling
+        after individual step completion.
+        """
+        try:
+            # Re-use sync logic but await the storage
+            integrated_data = self.get_integrated_data_sync(user_id, db)
+            await self._store_integrated_data(user_id, integrated_data, db)
+            logger.info(f"Refreshed integrated data (SSOT) for user {user_id}")
+        except Exception as e:
+            logger.error(f"Failed to refresh integrated data for user {user_id}: {e}")
+            # Non-blocking failure
+
+    async def store_competitive_sitemap_benchmarking(self, user_id: str, report: Dict[str, Any], db: Session) -> bool:
+        try:
+            if not user_id:
+                return False
+            if not isinstance(report, dict):
+                return False
+
+            session = db.query(OnboardingSession).filter(
+                OnboardingSession.user_id == user_id
+            ).order_by(OnboardingSession.updated_at.desc()).first()
+
+            if not session:
+                return False
+
+            website_analysis = db.query(WebsiteAnalysis).filter(
+                WebsiteAnalysis.session_id == session.id
+            ).order_by(WebsiteAnalysis.updated_at.desc()).first()
+
+            if not website_analysis:
+                return False
+
+            existing = website_analysis.seo_audit if isinstance(website_analysis.seo_audit, dict) else {}
+            existing["competitive_sitemap_benchmarking"] = report
+            website_analysis.seo_audit = existing
+            website_analysis.updated_at = datetime.utcnow()
+            
+            # Use flag_modified to ensure JSON update is detected by SQLAlchemy
+            from sqlalchemy.orm.attributes import flag_modified
+            flag_modified(website_analysis, "seo_audit")
+            
+            db.commit()
+
+            try:
+                await self.refresh_integrated_data(user_id, db)
+            except Exception:
+                pass
+
+            return True
+        except Exception as e:
+            logger.error(f"Failed to store competitive sitemap benchmarking for user {user_id}: {e}")
+            db.rollback()
+            return False
+
+    async def update_competitive_sitemap_benchmarking_status(self, user_id: str, status: str, db: Session, error: Optional[str] = None) -> bool:
+        """Update the status of the competitive sitemap benchmarking task."""
+        try:
+            if not user_id:
+                return False
+
+            session = db.query(OnboardingSession).filter(
+                OnboardingSession.user_id == user_id
+            ).order_by(OnboardingSession.updated_at.desc()).first()
+
+            if not session:
+                return False
+
+            website_analysis = db.query(WebsiteAnalysis).filter(
+                WebsiteAnalysis.session_id == session.id
+            ).order_by(WebsiteAnalysis.updated_at.desc()).first()
+
+            if not website_analysis:
+                return False
+
+            existing = website_analysis.seo_audit if isinstance(website_analysis.seo_audit, dict) else {}
+            
+            # Get existing benchmarking data or initialize
+            benchmarking = existing.get("competitive_sitemap_benchmarking", {})
+            if not isinstance(benchmarking, dict):
+                benchmarking = {}
+            
+            benchmarking["status"] = status
+            if error:
+                benchmarking["error"] = error
+            if status == "processing":
+                benchmarking["started_at"] = datetime.utcnow().isoformat()
+            
+            existing["competitive_sitemap_benchmarking"] = benchmarking
+            website_analysis.seo_audit = existing
+            # Force update flag if needed, but assignment should trigger it
+            website_analysis.updated_at = datetime.utcnow()
+            
+            # Use flag_modified if using JSON type with SQLAlchemy to ensure update
+            from sqlalchemy.orm.attributes import flag_modified
+            flag_modified(website_analysis, "seo_audit")
+            
+            db.commit()
+            return True
+        except Exception as e:
+            logger.error(f"Failed to update competitive sitemap benchmarking status for user {user_id}: {e}")
+            if db:
+                db.rollback()
+            return False
+
     async def process_onboarding_data(self, user_id: str, db: Session) -> Dict[str, Any]:
         """Process and integrate all onboarding data for a user.
         
@@ -49,6 +210,7 @@ class OnboardingDataIntegrationService:
             onboarding_session = self._get_onboarding_session(user_id, db)
             persona_data = self._get_persona_data(user_id, db)
             competitor_analysis = self._get_competitor_analysis(user_id, db)
+            deep_competitor_analysis = self._get_deep_competitor_analysis(user_id, db)
             gsc_analytics = await self._get_gsc_analytics(user_id)
             bing_analytics = await self._get_bing_analytics(user_id)
 
@@ -63,7 +225,15 @@ class OnboardingDataIntegrationService:
             logger.info(f"  - GSC Analytics: {'✅ Found' if gsc_analytics else '❌ Missing'}")
             logger.info(f"  - Bing Analytics: {'✅ Found' if bing_analytics else '❌ Missing'}")
 
-            # Process and integrate data
+            canonical_profile = self._build_canonical_profile(
+                website_analysis,
+                research_preferences,
+                persona_data,
+                onboarding_session,
+                competitor_analysis,
+                deep_competitor_analysis
+            )
+
             integrated_data = {
                 'website_analysis': website_analysis,
                 'research_preferences': research_preferences,
@@ -71,8 +241,10 @@ class OnboardingDataIntegrationService:
                 'onboarding_session': onboarding_session,
                 'persona_data': persona_data,
                 'competitor_analysis': competitor_analysis,
+                'deep_competitor_analysis': deep_competitor_analysis,
                 'gsc_analytics': gsc_analytics,
                 'bing_analytics': bing_analytics,
+                'canonical_profile': canonical_profile,
                 'data_quality': self._assess_data_quality(website_analysis, research_preferences, api_keys_data, persona_data, competitor_analysis, gsc_analytics, bing_analytics),
                 'processing_timestamp': datetime.utcnow().isoformat()
             }
@@ -105,7 +277,7 @@ class OnboardingDataIntegrationService:
             ).order_by(OnboardingSession.updated_at.desc()).first()
             
             if not session:
-                logger.warning(f"No onboarding session found for user {user_id}")
+                logger.info(f"No onboarding session found for user {user_id}")
                 return {}
             
             # Get the latest website analysis for this session
@@ -114,13 +286,17 @@ class OnboardingDataIntegrationService:
             ).order_by(WebsiteAnalysis.updated_at.desc()).first()
             
             if not website_analysis:
-                logger.warning(f"No website analysis found for user {user_id}")
+                logger.info(f"No website analysis found for user {user_id}")
                 return {}
             
             # Convert to dictionary and add metadata
             analysis_data = website_analysis.to_dict()
             analysis_data['data_freshness'] = self._calculate_freshness(website_analysis.updated_at)
             analysis_data['confidence_level'] = 0.9 if website_analysis.status == 'completed' else 0.5
+
+            site_url = website_analysis.website_url
+            if site_url:
+                analysis_data["full_site_seo_summary"] = self._get_full_site_seo_summary(user_id, site_url, db)
             
             logger.info(f"Retrieved website analysis for user {user_id}: {website_analysis.website_url}")
             return analysis_data
@@ -129,6 +305,36 @@ class OnboardingDataIntegrationService:
             logger.error(f"Error getting website analysis for user {user_id}: {str(e)}")
             return {}
 
+    def _get_full_site_seo_summary(self, user_id: str, website_url: str, db: Session) -> Dict[str, Any]:
+        try:
+            rows = db.query(SEOPageAudit).filter(
+                SEOPageAudit.user_id == user_id,
+                SEOPageAudit.website_url == website_url
+            ).all()
+
+            if not rows:
+                return {}
+
+            scored = [r for r in rows if r.overall_score is not None]
+            scores = [int(r.overall_score) for r in scored if isinstance(r.overall_score, (int, float))]
+            avg_score = round(sum(scores) / len(scores), 1) if scores else 0
+
+            fix_scheduled_count = len([r for r in scored if (r.status or "").lower() == "fix_scheduled"])
+
+            worst = sorted(scored, key=lambda r: r.overall_score if r.overall_score is not None else 10**9)[:5]
+            worst_pages = [{"page_url": r.page_url, "overall_score": r.overall_score, "status": r.status} for r in worst]
+
+            return {
+                "pages_audited": len(rows),
+                "pages_scored": len(scored),
+                "avg_score": avg_score,
+                "fix_scheduled_pages": fix_scheduled_count,
+                "worst_pages": worst_pages
+            }
+        except Exception as e:
+            logger.error(f"Error building full-site SEO summary for user {user_id}: {str(e)}")
+            return {}
+
     def _get_research_preferences(self, user_id: str, db: Session) -> Dict[str, Any]:
         """Get research preferences data for the user."""
         try:
@@ -138,7 +344,7 @@ class OnboardingDataIntegrationService:
             ).order_by(OnboardingSession.updated_at.desc()).first()
             
             if not session:
-                logger.warning(f"No onboarding session found for user {user_id}")
+                logger.info(f"No onboarding session found for user {user_id}")
                 return {}
             
             # Get research preferences for this session
@@ -147,7 +353,7 @@ class OnboardingDataIntegrationService:
             ).first()
             
             if not research_prefs:
-                logger.warning(f"No research preferences found for user {user_id}")
+                logger.info(f"No research preferences found for user {user_id}")
                 return {}
             
             # Convert to dictionary and add metadata
@@ -171,7 +377,7 @@ class OnboardingDataIntegrationService:
             ).order_by(OnboardingSession.updated_at.desc()).first()
             
             if not session:
-                logger.warning(f"No onboarding session found for user {user_id}")
+                logger.info(f"No onboarding session found for user {user_id}")
                 return {}
             
             # Get all API keys for this session
@@ -180,7 +386,7 @@ class OnboardingDataIntegrationService:
             ).all()
             
             if not api_keys:
-                logger.warning(f"No API keys found for user {user_id}")
+                logger.info(f"No API keys found for user {user_id}")
                 return {}
             
             # Convert to dictionary format
@@ -202,16 +408,14 @@ class OnboardingDataIntegrationService:
     def _get_onboarding_session(self, user_id: str, db: Session) -> Dict[str, Any]:
         """Get onboarding session data for the user."""
         try:
-            # Get the latest onboarding session for the user
             session = db.query(OnboardingSession).filter(
                 OnboardingSession.user_id == user_id
             ).order_by(OnboardingSession.updated_at.desc()).first()
             
             if not session:
-                logger.warning(f"No onboarding session found for user {user_id}")
+                logger.info(f"No onboarding session found for user {user_id}")
                 return {}
             
-            # Convert to dictionary
             session_data = {
                 'id': session.id,
                 'user_id': session.user_id,
@@ -225,11 +429,303 @@ class OnboardingDataIntegrationService:
             
             logger.info(f"Retrieved onboarding session for user {user_id}: step {session.current_step}, progress {session.progress}%")
             return session_data
-
+            
         except Exception as e:
             logger.error(f"Error getting onboarding session for user {user_id}: {str(e)}")
             return {}
 
+    def _build_canonical_profile(
+        self,
+        website_analysis: Dict[str, Any],
+        research_preferences: Dict[str, Any],
+        persona_data: Dict[str, Any],
+        onboarding_session: Dict[str, Any],
+        competitor_analysis: List[Dict[str, Any]],
+        deep_competitor_analysis: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        try:
+            core_persona = None
+            if persona_data:
+                if isinstance(persona_data, dict):
+                    core_persona = persona_data.get('corePersona') or persona_data.get('core_persona')
+
+            website_target = {}
+            if website_analysis and isinstance(website_analysis, dict):
+                value = website_analysis.get('target_audience') or {}
+                if isinstance(value, dict):
+                    website_target = value
+
+            research_target = {}
+            if research_preferences and isinstance(research_preferences, dict):
+                value = research_preferences.get('target_audience') or {}
+                if isinstance(value, dict):
+                    research_target = value
+
+            industry = None
+            if core_persona and isinstance(core_persona, dict):
+                value = core_persona.get('industry')
+                if value:
+                    industry = value
+            if not industry and website_target:
+                value = website_target.get('industry_focus')
+                if value:
+                    industry = value
+            if not industry and research_target:
+                value = research_target.get('industry_focus')
+                if value:
+                    industry = value
+
+            target_audience = None
+            target_source = None
+            if core_persona and isinstance(core_persona, dict):
+                value = core_persona.get('target_audience')
+                if value:
+                    target_audience = value
+                    target_source = 'persona_core'
+            if not target_audience and website_target:
+                value = website_target.get('demographics') or website_target.get('target_audience')
+                if value:
+                    target_audience = value
+                    target_source = 'website_analysis'
+            if not target_audience and research_target:
+                value = research_target.get('demographics') or research_target.get('target_audience')
+                if value:
+                    target_audience = value
+                    target_source = 'research_preferences'
+
+            writing_style = {}
+            if website_analysis and isinstance(website_analysis, dict):
+                value = website_analysis.get('writing_style')
+                if isinstance(value, dict):
+                    writing_style = value
+            if not writing_style and research_preferences and isinstance(research_preferences, dict):
+                value = research_preferences.get('writing_style')
+                if isinstance(value, dict):
+                    writing_style = value
+
+            writing_tone = None
+            writing_voice = None
+            writing_complexity = None
+            writing_engagement = None
+            writing_source = None
+            if writing_style:
+                value = writing_style.get('tone')
+                if value:
+                    writing_tone = value
+                
+                value = writing_style.get('voice')
+                if value:
+                    writing_voice = value
+
+                value = writing_style.get('complexity')
+                if value:
+                    writing_complexity = value
+
+                value = writing_style.get('engagement_level')
+                if value:
+                    writing_engagement = value
+
+                if website_analysis and website_analysis.get('writing_style'):
+                    writing_source = 'website_analysis'
+                elif research_preferences and research_preferences.get('writing_style'):
+                    writing_source = 'research_preferences'
+
+            # Brand & Visual Identity
+            brand_colors = []
+            brand_values = []
+            visual_style = {}
+            brand_source = None
+            
+            if website_analysis and isinstance(website_analysis, dict):
+                brand_analysis = website_analysis.get('brand_analysis', {})
+                if brand_analysis:
+                    brand_colors = brand_analysis.get('color_palette', [])
+                    brand_values = brand_analysis.get('brand_values', [])
+                    brand_source = 'website_analysis'
+                
+                style_guidelines = website_analysis.get('style_guidelines', {})
+                if style_guidelines:
+                    visual_style = {
+                        'aesthetic': style_guidelines.get('aesthetic'),
+                        'visual_style': style_guidelines.get('visual_style')
+                    }
+
+            # Content Strategy Insights
+            strategy_insights = {}
+            if website_analysis and isinstance(website_analysis, dict):
+                strategy_insights = website_analysis.get('content_strategy_insights', {})
+
+            seo_profile: Dict[str, Any] = {}
+            if website_analysis and isinstance(website_analysis, dict):
+                seo_profile["homepage_seo_audit"] = website_analysis.get("seo_audit") or {}
+                seo_profile["full_site_seo_summary"] = website_analysis.get("full_site_seo_summary") or {}
+                sitemap_strategy = website_analysis.get("sitemap_strategy_insights")
+                if sitemap_strategy:
+                    seo_profile["sitemap_strategy_insights"] = sitemap_strategy
+
+            competitor_seo_benchmarks = self._build_competitor_seo_benchmarks(competitor_analysis)
+            if competitor_seo_benchmarks:
+                seo_profile["competitor_seo_benchmarks"] = competitor_seo_benchmarks
+
+            # Platform Preferences
+            platform_preferences = []
+            platform_source = None
+            
+            if core_persona and isinstance(core_persona, dict):
+                # Check persona_data for platforms
+                if isinstance(persona_data, dict):
+                    selected = persona_data.get('selectedPlatforms')
+                    if selected:
+                        platform_preferences = selected
+                        platform_source = 'persona_data'
+                    else:
+                        platform_personas = persona_data.get('platformPersonas')
+                        if platform_personas:
+                            platform_preferences = list(platform_personas.keys())
+                            platform_source = 'persona_data'
+
+            content_types = []
+            content_source = None
+            if research_preferences and isinstance(research_preferences, dict):
+                prefs_content = research_preferences.get('content_types')
+                if isinstance(prefs_content, list):
+                    content_types = list(prefs_content)
+                    if content_types:
+                        content_source = 'research_preferences'
+            if not content_types and website_analysis and isinstance(website_analysis, dict):
+                content_type_data = website_analysis.get('content_type') or {}
+                if isinstance(content_type_data, dict):
+                    primary = content_type_data.get('primary_type')
+                    if primary:
+                        content_types.append(primary)
+                    secondary = content_type_data.get('secondary_types')
+                    if isinstance(secondary, list):
+                        content_types.extend(secondary)
+                    if content_types:
+                        content_source = 'website_analysis'
+
+            research_depth = None
+            auto_research = None
+            factual_content = None
+            if research_preferences and isinstance(research_preferences, dict):
+                research_depth = research_preferences.get('research_depth')
+                auto_research = research_preferences.get('auto_research')
+                factual_content = research_preferences.get('factual_content')
+
+            business_info = {}
+            if industry:
+                business_info['industry'] = industry
+            if target_audience:
+                business_info['target_audience'] = target_audience
+
+            sources = {
+                'industry': None,
+                'target_audience': target_source,
+                'writing_tone': writing_source,
+                'content_types': content_source,
+                'brand_identity': brand_source,
+                'platform_preferences': platform_source,
+                'seo_profile': 'website_analysis' if website_analysis else None
+            }
+            if core_persona and isinstance(core_persona, dict) and core_persona.get('industry'):
+                sources['industry'] = 'persona_core'
+            elif website_target.get('industry_focus'):
+                sources['industry'] = 'website_analysis'
+            elif research_target.get('industry_focus'):
+                sources['industry'] = 'research_preferences'
+
+            competitive_sitemap_benchmarking = {}
+            try:
+                if website_analysis and isinstance(website_analysis, dict):
+                    seo_audit = website_analysis.get("seo_audit")
+                    if isinstance(seo_audit, dict):
+                        report = seo_audit.get("competitive_sitemap_benchmarking")
+                        if isinstance(report, dict):
+                            benchmark = report.get("benchmark") if isinstance(report.get("benchmark"), dict) else {}
+                            gaps = benchmark.get("gaps") if isinstance(benchmark.get("gaps"), dict) else {}
+                            missing_sections = gaps.get("missing_sections") if isinstance(gaps.get("missing_sections"), list) else []
+                            competitive_sitemap_benchmarking = {
+                                "status": "available",
+                                "last_run": report.get("timestamp") or report.get("analysis_date"),
+                                "competitors_analyzed": benchmark.get("competitors_analyzed"),
+                                "missing_sections_count": len(missing_sections)
+                            }
+            except Exception:
+                competitive_sitemap_benchmarking = {}
+
+            competitive_intelligence = {
+                'deep_competitor_analysis': deep_competitor_analysis or {},
+                'competitive_sitemap_benchmarking': competitive_sitemap_benchmarking,
+                'strategic_insights_history': website_analysis.get("strategic_insights_history", []) if isinstance(website_analysis, dict) else []
+            }
+
+            return {
+                'industry': industry,
+                'target_audience': target_audience,
+                'writing_tone': writing_tone or 'professional',
+                'writing_voice': writing_voice or 'authoritative',
+                'writing_complexity': writing_complexity or 'intermediate',
+                'writing_engagement': writing_engagement or 'moderate',
+                'content_types': content_types,
+                'brand_colors': brand_colors,
+                'brand_values': brand_values,
+                'visual_style': visual_style,
+                'strategy_insights': strategy_insights,
+                'seo_profile': seo_profile,
+                'competitive_intelligence': competitive_intelligence,
+                'platform_preferences': platform_preferences,
+                'research_depth': research_depth,
+                'auto_research': auto_research,
+                'factual_content': factual_content,
+                'business_info': business_info,
+                'sources': sources
+            }
+        except Exception as e:
+            logger.error(f"Error building canonical profile: {str(e)}")
+            return {}
+
+    def _build_competitor_seo_benchmarks(self, competitor_analysis: List[Dict[str, Any]]) -> Dict[str, Any]:
+        try:
+            if not competitor_analysis:
+                return {}
+
+            rows = []
+            for comp in competitor_analysis:
+                analysis_data = comp.get("analysis_data") if isinstance(comp, dict) else None
+                if not isinstance(analysis_data, dict):
+                    continue
+                seo_audit = analysis_data.get("seo_audit")
+                if not isinstance(seo_audit, dict):
+                    continue
+                score = seo_audit.get("overall_score")
+                if score is None:
+                    continue
+                rows.append({
+                    "competitor_url": comp.get("competitor_url") or comp.get("url") or comp.get("website_url"),
+                    "competitor_domain": comp.get("competitor_domain") or comp.get("domain"),
+                    "overall_score": score,
+                    "last_analyzed_at": comp.get("updated_at") or comp.get("analysis_date")
+                })
+
+            if not rows:
+                return {}
+
+            scores = [r["overall_score"] for r in rows if isinstance(r.get("overall_score"), (int, float))]
+            avg_score = round(sum(scores) / len(scores), 1) if scores else None
+
+            best = max(rows, key=lambda r: r.get("overall_score") or 0)
+            worst = min(rows, key=lambda r: r.get("overall_score") or 0)
+
+            return {
+                "competitors_with_seo_audit": len(rows),
+                "avg_homepage_seo_score": avg_score,
+                "best_competitor": best,
+                "worst_competitor": worst
+            }
+        except Exception as e:
+            logger.error(f"Error building competitor SEO benchmarks: {str(e)}")
+            return {}
+
     def _assess_data_quality(self, website_analysis: Dict, research_preferences: Dict, api_keys_data: Dict, persona_data: Dict = None, competitor_analysis: List = None, gsc_analytics: Dict = None, bing_analytics: Dict = None) -> Dict[str, Any]:
         """Assess the quality and completeness of onboarding data."""
         try:
@@ -432,7 +928,7 @@ class OnboardingDataIntegrationService:
             ).first()
             
             if not persona:
-                logger.warning(f"No persona data found for user {user_id}")
+                logger.info(f"[Persona] No persona data found for user {user_id}")
                 return {}
             
             # Convert to dictionary and add metadata
@@ -456,10 +952,10 @@ class OnboardingDataIntegrationService:
             ).order_by(OnboardingSession.updated_at.desc()).first()
             
             if not session:
-                logger.warning(f"🔍 COMPETITOR VALIDATION: No onboarding session found for user {user_id}")
+                logger.info(f"[CompetitorAnalysis] No onboarding session found for user {user_id}")
                 return []
             
-            logger.warning(f"🔍 COMPETITOR VALIDATION: Found session {session.id} for user {user_id}")
+            logger.info(f"[CompetitorAnalysis] user={user_id} session={session.id} (latest)")
             
             # Get all competitor analyses for this session
             competitor_records = db.query(CompetitorAnalysis).filter(
@@ -467,22 +963,10 @@ class OnboardingDataIntegrationService:
             ).order_by(CompetitorAnalysis.updated_at.desc()).all()
             
             if not competitor_records:
-                logger.warning(f"🔍 COMPETITOR VALIDATION: No competitor analysis records found for user {user_id}, session {session.id}")
-                logger.warning(f"  Checking all sessions for user {user_id}...")
-                # Check all sessions for this user
-                all_sessions = db.query(OnboardingSession).filter(
-                    OnboardingSession.user_id == user_id
-                ).all()
-                logger.warning(f"  Total sessions for user: {len(all_sessions)}")
-                for sess in all_sessions:
-                    comp_count = db.query(CompetitorAnalysis).filter(
-                        CompetitorAnalysis.session_id == sess.id
-                    ).count()
-                    session_timestamp = getattr(sess, 'started_at', None) or getattr(sess, 'updated_at', None)
-                    logger.warning(f"  Session {sess.id} (timestamp: {session_timestamp}): {comp_count} competitors")
+                logger.info(f"[CompetitorAnalysis] No competitor records found for user={user_id} session={session.id}")
                 return []
             
-            logger.warning(f"🔍 COMPETITOR VALIDATION: Found {len(competitor_records)} competitor records for user {user_id}")
+            logger.info(f"[CompetitorAnalysis] session={session.id} records={len(competitor_records)} user={user_id}")
             
             # Convert to list of dictionaries
             # Use to_dict() which includes competitor_url, competitor_domain, analysis_data
@@ -496,25 +980,68 @@ class OnboardingDataIntegrationService:
                 competitor_dict['confidence_level'] = 0.9 if record.status == 'completed' else 0.5
                 competitors.append(competitor_dict)
             
-            logger.info(f"Retrieved {len(competitors)} competitor analyses for user {user_id}")
+            logger.info(f"[CompetitorAnalysis] retrieved={len(competitors)} user={user_id}")
             if competitors:
-                logger.warning(f"🔍 Sample competitor keys: {list(competitors[0].keys())}")
-                logger.warning(f"🔍 Sample competitor has analysis_data: {'analysis_data' in competitors[0]}")
-                if 'analysis_data' in competitors[0]:
-                    logger.warning(f"🔍 Sample analysis_data keys: {list(competitors[0]['analysis_data'].keys()) if isinstance(competitors[0]['analysis_data'], dict) else 'Not a dict'}")
+                try:
+                    sample = competitors[0]
+                    logger.debug(f"[CompetitorAnalysis] sample_keys={list(sample.keys())} has_analysis_data={'analysis_data' in sample}")
+                    if isinstance(sample.get('analysis_data'), dict):
+                        logger.debug(f"[CompetitorAnalysis] analysis_data_keys={list(sample['analysis_data'].keys())}")
+                except Exception:
+                    pass
             return competitors
 
         except Exception as e:
             logger.error(f"Error getting competitor analysis for user {user_id}: {str(e)}")
             return []
 
+    def _get_deep_competitor_analysis(self, user_id: str, db: Session) -> Dict[str, Any]:
+        try:
+            task = db.query(DeepCompetitorAnalysisTask).filter(
+                DeepCompetitorAnalysisTask.user_id == user_id
+            ).order_by(DeepCompetitorAnalysisTask.updated_at.desc()).first()
+
+            if not task:
+                return {
+                    "status": "not_scheduled",
+                    "last_run": None,
+                    "report": None
+                }
+
+            latest_log = db.query(DeepCompetitorAnalysisExecutionLog).filter(
+                DeepCompetitorAnalysisExecutionLog.task_id == task.id
+            ).order_by(DeepCompetitorAnalysisExecutionLog.execution_date.desc()).first()
+
+            last_run = None
+            if latest_log and latest_log.execution_date:
+                last_run = latest_log.execution_date.isoformat()
+
+            report = None
+            if latest_log and latest_log.status == "success":
+                report = latest_log.result_data
+
+            payload = task.payload if isinstance(task.payload, dict) else {}
+            competitors = payload.get("competitors") if isinstance(payload, dict) else None
+
+            return {
+                "status": task.status,
+                "next_execution": task.next_execution.isoformat() if task.next_execution else None,
+                "last_run": last_run,
+                "last_status": latest_log.status if latest_log else None,
+                "competitors_count": len(competitors) if isinstance(competitors, list) else None,
+                "report": report
+            }
+        except Exception as e:
+            logger.error(f"Error getting deep competitor analysis for user {user_id}: {str(e)}")
+            return {}
+
     async def _get_gsc_analytics(self, user_id: str) -> Dict[str, Any]:
         """Get Google Search Console analytics data for the user."""
         try:
             from services.seo.dashboard_service import SEODashboardService
             from services.database import get_db_session
             
-            db = get_db_session()
+            db = get_db_session(user_id)
             try:
                 dashboard_service = SEODashboardService(db)
                 gsc_data = await dashboard_service.get_gsc_data(user_id)
@@ -545,7 +1072,7 @@ class OnboardingDataIntegrationService:
             from services.bing_analytics_storage_service import BingAnalyticsStorageService
             from services.database import get_db_session
             
-            db = get_db_session()
+            db = get_db_session(user_id)
             try:
                 dashboard_service = SEODashboardService(db)
                 bing_data = await dashboard_service.get_bing_data(user_id)
@@ -553,13 +1080,15 @@ class OnboardingDataIntegrationService:
                 db.close()
             
             # Also try to get from storage service for more detailed metrics
-            bing_storage = BingAnalyticsStorageService(os.getenv('DATABASE_URL', 'sqlite:///alwrity.db'))
+            from services.database import get_user_db_path
+            db_path = get_user_db_path(user_id)
+            bing_storage = BingAnalyticsStorageService(f'sqlite:///{db_path}')
             
             # Get site URL from onboarding session if available
             site_url = None
             try:
                 from services.database import get_db_session
-                with get_db_session() as db:
+                with get_db_session(user_id) as db:
                     session = db.query(OnboardingSession).filter(
                         OnboardingSession.user_id == user_id
                     ).order_by(OnboardingSession.updated_at.desc()).first()
@@ -663,4 +1192,4 @@ class OnboardingDataIntegrationService:
 
         except Exception as e:
             logger.error(f"Error getting integrated data for user {user_id}: {str(e)}")
-            return None 
+            return None 

backend/api/content_planning/services/content_strategy/utils/data_processors.py

@@ -195,14 +195,29 @@ class DataProcessorService:
         }
         
         # Competitive Intelligence Fields
+        # Extract competitors from competitor_analysis list in processed_data
+        competitors_list = processed_data.get('competitor_analysis', [])
+        competitor_names = []
+        
+        if competitors_list:
+            for comp in competitors_list:
+                # Try to get domain or title, fallback to URL
+                name = comp.get('competitor_domain') or comp.get('domain') or comp.get('title') or comp.get('competitor_url') or comp.get('url')
+                if name:
+                    competitor_names.append(name)
+        
+        # Fallback to website_analysis competitors if available (legacy/manual entry)
+        if not competitor_names and website_data.get('competitors'):
+            competitor_names = website_data.get('competitors')
+            
         fields['top_competitors'] = {
-            'value': website_data.get('competitors', [
+            'value': competitor_names if competitor_names else [
                 'Competitor A - Industry Leader',
                 'Competitor B - Emerging Player', 
                 'Competitor C - Niche Specialist'
-            ]),
-            'source': 'website_analysis',
-            'confidence': website_data.get('confidence_level', 0.8)
+            ],
+            'source': 'competitor_analysis' if competitors_list else ('website_analysis' if website_data.get('competitors') else 'default'),
+            'confidence': 0.9 if competitors_list else (website_data.get('confidence_level', 0.8) if website_data.get('competitors') else 0.3)
         }
         
         fields['competitor_content_strategies'] = {

backend/api/content_planning/services/enhanced_strategy_db_service.py

@@ -22,7 +22,7 @@ class EnhancedStrategyDBService:
     def __init__(self, db: Session):
         self.db = db
     
-    async def get_enhanced_strategy(self, strategy_id: int, user_id: Optional[int] = None) -> Optional[EnhancedContentStrategy]:
+    async def get_enhanced_strategy(self, strategy_id: int, user_id: Optional[str] = None) -> Optional[EnhancedContentStrategy]:
         """
         Get an enhanced strategy by ID.
         
@@ -54,7 +54,7 @@ class EnhancedStrategyDBService:
             logger.error(f"Error getting enhanced strategy {strategy_id}: {str(e)}")
             return None
     
-    async def get_enhanced_strategies(self, user_id: Optional[int] = None, strategy_id: Optional[int] = None) -> List[EnhancedContentStrategy]:
+    async def get_enhanced_strategies(self, user_id: Optional[str] = None, strategy_id: Optional[int] = None) -> List[EnhancedContentStrategy]:
         """Get enhanced strategies with optional filtering."""
         try:
             query = self.db.query(EnhancedContentStrategy)
@@ -183,7 +183,7 @@ class EnhancedStrategyDBService:
             logger.error(f"Error getting onboarding integration for strategy {strategy_id}: {str(e)}")
             return None
     
-    async def get_strategy_completion_stats(self, user_id: int) -> Dict[str, Any]:
+    async def get_strategy_completion_stats(self, user_id: str) -> Dict[str, Any]:
         """Get completion statistics for all strategies of a user."""
         try:
             strategies = await self.get_enhanced_strategies(user_id=user_id)
@@ -207,7 +207,7 @@ class EnhancedStrategyDBService:
                 'user_id': user_id
             }
     
-    async def search_enhanced_strategies(self, user_id: int, search_term: str) -> List[EnhancedContentStrategy]:
+    async def search_enhanced_strategies(self, user_id: str, search_term: str) -> List[EnhancedContentStrategy]:
         """Search enhanced strategies by name or industry."""
         try:
             return self.db.query(EnhancedContentStrategy).filter(
@@ -256,7 +256,7 @@ class EnhancedStrategyDBService:
             logger.error(f"Error getting strategy export data for strategy {strategy_id}: {str(e)}")
             return None 
 
-    async def save_autofill_insights(self, *, strategy_id: int, user_id: int, payload: Dict[str, Any]) -> Optional[ContentStrategyAutofillInsights]:
+    async def save_autofill_insights(self, *, strategy_id: int, user_id: str, payload: Dict[str, Any]) -> Optional[ContentStrategyAutofillInsights]:
         """Persist accepted auto-fill inputs used to create a strategy."""
         try:
             record = ContentStrategyAutofillInsights(
@@ -300,4 +300,4 @@ class EnhancedStrategyDBService:
             }
         except Exception as e:
             logger.error(f"Error fetching latest autofill insights for strategy {strategy_id}: {str(e)}")
-            return None 
+            return None 

backend/api/content_planning/services/enhanced_strategy_service.py

@@ -64,11 +64,11 @@ class EnhancedStrategyService:
         """Create a new enhanced content strategy - delegates to core service."""
         return await self.core_service.create_enhanced_strategy(strategy_data, db)
     
-    async def get_enhanced_strategies(self, user_id: Optional[int] = None, strategy_id: Optional[int] = None, db: Session = None) -> Dict[str, Any]:
+    async def get_enhanced_strategies(self, user_id: Optional[str] = None, strategy_id: Optional[int] = None, db: Session = None) -> Dict[str, Any]:
         """Get enhanced content strategies - delegates to core service."""
         return await self.core_service.get_enhanced_strategies(user_id, strategy_id, db)
 
-    async def _enhance_strategy_with_onboarding_data(self, strategy: Any, user_id: int, db: Session) -> None:
+    async def _enhance_strategy_with_onboarding_data(self, strategy: Any, user_id: str, db: Session) -> None:
         """Enhance strategy with onboarding data - delegates to core service."""
         return await self.core_service._enhance_strategy_with_onboarding_data(strategy, user_id, db)
 
@@ -239,4 +239,4 @@ class EnhancedStrategyService:
     def _initialize_caches(self) -> None:
         """Initialize caches - delegates to core service."""
         # This is now handled by the core service
-        pass
+        pass

backend/api/content_planning/services/gap_analysis_service.py

@@ -11,7 +11,8 @@ from sqlalchemy.orm import Session
 # Import database services
 from services.content_planning_db import ContentPlanningDBService
 from services.ai_analysis_db_service import AIAnalysisDBService
-from services.onboarding.data_service import OnboardingDataService
+from services.database import SessionLocal, get_session_for_user
+from api.content_planning.services.content_strategy.onboarding import OnboardingDataIntegrationService
 
 # Import migrated content gap analysis services
 from services.content_gap_analyzer.content_gap_analyzer import ContentGapAnalyzer
@@ -30,7 +31,7 @@ class GapAnalysisService:
     
     def __init__(self):
         self.ai_analysis_db_service = AIAnalysisDBService()
-        self.onboarding_service = OnboardingDataService()
+        self.onboarding_integration_service = OnboardingDataIntegrationService()
         
         # Initialize migrated services
         self.content_gap_analyzer = ContentGapAnalyzer()
@@ -57,13 +58,13 @@ class GapAnalysisService:
             logger.error(f"Error creating content gap analysis: {str(e)}")
             raise ContentPlanningErrorHandler.handle_general_error(e, "create_gap_analysis")
     
-    async def get_gap_analyses(self, user_id: Optional[int] = None, strategy_id: Optional[int] = None, force_refresh: bool = False) -> Dict[str, Any]:
+    async def get_gap_analyses(self, user_id: Optional[Any] = None, strategy_id: Optional[int] = None, force_refresh: bool = False) -> Dict[str, Any]:
         """Get content gap analysis with real AI insights - Database first approach."""
         try:
             logger.info(f"🚀 Starting content gap analysis for user: {user_id}, strategy: {strategy_id}, force_refresh: {force_refresh}")
             
             # Use user_id or default to 1
-            current_user_id = user_id or 1
+            current_user_id = user_id or "1"
             
             # Skip database check if force_refresh is True
             if not force_refresh:
@@ -93,13 +94,17 @@ class GapAnalysisService:
             # No recent analysis found or force refresh requested, run new AI analysis
             logger.info(f"🔄 Running new gap analysis for user {current_user_id} (force_refresh: {force_refresh})")
             
-            # Get personalized inputs from onboarding data
-            personalized_inputs = self.onboarding_service.get_personalized_ai_inputs(current_user_id)
+            # Get personalized inputs from onboarding data (SSOT)
+            db = get_session_for_user(str(current_user_id))
+            try:
+                personalized_inputs = await self.onboarding_integration_service.process_onboarding_data(str(current_user_id), db)
+            finally:
+                db.close()
             
             logger.info(f"📊 Using personalized inputs: {len(personalized_inputs)} data points")
             
             # Generate real AI-powered gap analysis
-            gap_analysis = await self.ai_engine_service.generate_content_recommendations(personalized_inputs)
+            gap_analysis = await self.ai_engine_service.generate_content_recommendations(personalized_inputs, user_id=str(current_user_id))
             
             logger.info(f"✅ AI gap analysis completed: {len(gap_analysis)} recommendations")
             
@@ -148,67 +153,34 @@ class GapAnalysisService:
             logger.error(f"Error getting content gap analysis: {str(e)}")
             raise ContentPlanningErrorHandler.handle_general_error(e, "get_gap_analysis_by_id")
     
-    async def analyze_content_gaps(self, request_data: Dict[str, Any]) -> Dict[str, Any]:
+    async def analyze_content_gaps(self, request_data: Dict[str, Any], user_id: str) -> Dict[str, Any]:
         """Analyze content gaps between your website and competitors."""
         try:
             logger.info(f"Starting content gap analysis for: {request_data.get('website_url', 'Unknown')}")
             
-            # Use migrated services for actual analysis
-            analysis_results = {}
-            
-            # 1. Website Analysis
-            logger.info("Performing website analysis...")
-            website_analysis = await self.website_analyzer.analyze_website_content(request_data.get('website_url'))
-            analysis_results['website_analysis'] = website_analysis
-            
-            # 2. Competitor Analysis
-            logger.info("Performing competitor analysis...")
-            competitor_analysis = await self.competitor_analyzer.analyze_competitors(request_data.get('competitor_urls', []))
-            analysis_results['competitor_analysis'] = competitor_analysis
-            
-            # 3. Keyword Research
-            logger.info("Performing keyword research...")
-            keyword_analysis = await self.keyword_researcher.research_keywords(
-                industry=request_data.get('industry'),
-                target_keywords=request_data.get('target_keywords')
-            )
-            analysis_results['keyword_analysis'] = keyword_analysis
-            
-            # 4. Content Gap Analysis
-            logger.info("Performing content gap analysis...")
-            gap_analysis = await self.content_gap_analyzer.identify_content_gaps(
-                website_url=request_data.get('website_url'),
+            # Use ContentGapAnalyzer for comprehensive analysis
+            results = await self.content_gap_analyzer.analyze_comprehensive_gap(
+                target_url=request_data.get('website_url'),
                 competitor_urls=request_data.get('competitor_urls', []),
-                keyword_data=keyword_analysis
+                target_keywords=request_data.get('target_keywords', []),
+                user_id=user_id,
+                industry=request_data.get('industry', 'general')
             )
-            analysis_results['gap_analysis'] = gap_analysis
             
-            # 5. AI-Powered Recommendations
-            logger.info("Generating AI recommendations...")
-            recommendations = await self.ai_engine_service.generate_recommendations(
-                website_analysis=website_analysis,
-                competitor_analysis=competitor_analysis,
-                gap_analysis=gap_analysis,
-                keyword_analysis=keyword_analysis
-            )
-            analysis_results['recommendations'] = recommendations
+            if 'error' in results:
+                raise Exception(results['error'])
             
-            # 6. Strategic Opportunities
-            logger.info("Identifying strategic opportunities...")
-            opportunities = await self.ai_engine_service.identify_strategic_opportunities(
-                gap_analysis=gap_analysis,
-                competitor_analysis=competitor_analysis,
-                keyword_analysis=keyword_analysis
-            )
-            analysis_results['opportunities'] = opportunities
-            
-            # Prepare response
+            # Map results to ContentGapAnalysisFullResponse structure
+            # ContentGapAnalyzer returns a rich structure, we map it to the response model
             response_data = {
-                'website_analysis': analysis_results['website_analysis'],
-                'competitor_analysis': analysis_results['competitor_analysis'],
-                'gap_analysis': analysis_results['gap_analysis'],
-                'recommendations': analysis_results['recommendations'],
-                'opportunities': analysis_results['opportunities'],
+                'website_analysis': {
+                    'serp_analysis': results.get('serp_analysis', {}),
+                    'keyword_expansion': results.get('keyword_expansion', {})
+                },
+                'competitor_analysis': results.get('competitor_content', {}),
+                'gap_analysis': results.get('gap_analysis', {}),
+                'recommendations': results.get('recommendations', []),
+                'opportunities': results.get('ai_insights', {}).get('strategic_insights', []),
                 'created_at': datetime.utcnow()
             }