Add brand analysis columns to onboarding database and migration scripts

backend/api/onboarding_utils/onboarding_completion_service.py

@@ -8,13 +8,16 @@ from fastapi import HTTPException
 from loguru import logger
 
 from services.api_key_manager import get_onboarding_progress_for_user, get_api_key_manager, StepStatus
+from services.onboarding_database_service import OnboardingDatabaseService
+from services.database import get_db
 from services.persona_analysis_service import PersonaAnalysisService
 
 class OnboardingCompletionService:
     """Service for handling onboarding completion logic."""
     
     def __init__(self):
-        self.required_steps = [1, 2, 3, 6]  # Steps 1, 2, 3, and 6 are required
+        # Only pre-requisite steps; step 6 is the finalization itself
+        self.required_steps = [1, 2, 3]
     
     async def complete_onboarding(self, current_user: Dict[str, Any]) -> Dict[str, Any]:
         """Complete the onboarding process with full validation."""
@@ -22,8 +25,8 @@ class OnboardingCompletionService:
             user_id = str(current_user.get('id'))
             progress = get_onboarding_progress_for_user(user_id)
             
-            # Validate required steps are completed
-            missing_steps = self._validate_required_steps(progress)
+            # Validate required steps are completed (with DB-aware fallbacks)
+            missing_steps = self._validate_required_steps(user_id, progress)
             if missing_steps:
                 missing_steps_str = ", ".join(missing_steps)
                 raise HTTPException(
@@ -53,13 +56,75 @@ class OnboardingCompletionService:
             logger.error(f"Error completing onboarding: {str(e)}")
             raise HTTPException(status_code=500, detail="Internal server error")
     
-    def _validate_required_steps(self, progress) -> List[str]:
-        """Validate that all required steps are completed."""
+    def _validate_required_steps(self, user_id: str, progress) -> List[str]:
+        """Validate that all required steps are completed.
+
+        This method trusts the progress tracker, but also falls back to
+        database presence for Steps 2 and 3 so migration from file→DB
+        does not block completion.
+        """
         missing_steps = []
-        
+        db = None
+        db_service = None
+        try:
+            db = next(get_db())
+            db_service = OnboardingDatabaseService(db)
+        except Exception:
+            db = None
+            db_service = None
+
         for step_num in self.required_steps:
             step = progress.get_step_data(step_num)
-            if step and step.status not in [StepStatus.COMPLETED, StepStatus.SKIPPED]:
+            if step and step.status in [StepStatus.COMPLETED, StepStatus.SKIPPED]:
+                continue
+
+            # DB-aware fallbacks for migration period
+            try:
+                if db_service:
+                    if step_num == 2:
+                        # Treat as completed if website analysis exists in DB
+                        website = db_service.get_website_analysis(user_id, db)
+                        if website and (website.get('website_url') or website.get('writing_style')):
+                            # Optionally mark as completed in progress to keep state consistent
+                            try:
+                                progress.mark_step_completed(2, {'source': 'db-fallback'})
+                            except Exception:
+                                pass
+                            continue
+                        # Secondary fallback: research preferences captured style data
+                        prefs = db_service.get_research_preferences(user_id, db)
+                        if prefs and (prefs.get('writing_style') or prefs.get('content_characteristics')):
+                            try:
+                                progress.mark_step_completed(2, {'source': 'research-prefs-fallback'})
+                            except Exception:
+                                pass
+                            continue
+                        # Tertiary fallback: persona data created implies earlier steps done
+                        persona = None
+                        try:
+                            persona = db_service.get_persona_data(user_id, db)
+                        except Exception:
+                            persona = None
+                        if persona and persona.get('corePersona'):
+                            try:
+                                progress.mark_step_completed(2, {'source': 'persona-fallback'})
+                            except Exception:
+                                pass
+                            continue
+                    if step_num == 3:
+                        # Treat as completed if research preferences exist in DB
+                        prefs = db_service.get_research_preferences(user_id, db)
+                        if prefs and prefs.get('research_depth'):
+                            try:
+                                progress.mark_step_completed(3, {'source': 'db-fallback'})
+                            except Exception:
+                                pass
+                            continue
+            except Exception:
+                # If DB check fails, fall back to progress status only
+                pass
+
+            if step:
                 missing_steps.append(step.title)
         
         return missing_steps

backend/api/onboarding_utils/onboarding_summary_service.py

@@ -9,6 +9,7 @@ from loguru import logger
 
 from services.api_key_manager import get_api_key_manager
 from services.database import get_db
+from services.onboarding_database_service import OnboardingDatabaseService
 from services.website_analysis_service import WebsiteAnalysisService
 from services.research_preferences_service import ResearchPreferencesService
 from services.persona_analysis_service import PersonaAnalysisService
@@ -23,14 +24,10 @@ class OnboardingSummaryService:
         Args:
             user_id: Clerk user ID from authenticated request
         """
-        # Convert Clerk user ID to integer for database compatibility
-        try:
-            self.user_id_int = int(user_id.replace('user_', '').replace('-', '')[:8], 16) % 2147483647
-        except:
-            self.user_id_int = hash(user_id) % 2147483647
+        self.user_id = user_id  # Store Clerk user ID (string)
+        self.db_service = OnboardingDatabaseService()
         
-        self.user_id = user_id  # Store original Clerk ID for logging
-        self.session_id = self.user_id_int  # Use user ID as session ID for backwards compatibility
+        logger.info(f"OnboardingSummaryService initialized for user {user_id} (database mode)")
     
     async def get_onboarding_summary(self) -> Dict[str, Any]:
         """Get comprehensive onboarding summary for FinalStep."""
@@ -69,40 +66,75 @@ class OnboardingSummaryService:
             raise HTTPException(status_code=500, detail="Internal server error")
     
     def _get_api_keys(self) -> Dict[str, Any]:
-        """Get configured API keys."""
-        api_manager = get_api_key_manager()
-        return api_manager.get_all_keys()
-    
-    def _get_website_analysis(self) -> Optional[Dict[str, Any]]:
-        """Get website analysis data."""
+        """Get configured API keys from database."""
         try:
             db = next(get_db())
-            website_service = WebsiteAnalysisService(db)
-            return website_service.get_analysis_by_session(self.session_id)
+            api_keys = self.db_service.get_api_keys(self.user_id, db)
+            logger.info(f"Retrieved {len(api_keys)} API keys from database for user {self.user_id}")
+            return api_keys
         except Exception as e:
-            logger.warning(f"Could not get website analysis: {str(e)}")
+            logger.error(f"Error getting API keys from database: {e}")
+            return {}
+    
+    def _get_website_analysis(self) -> Optional[Dict[str, Any]]:
+        """Get website analysis data from database (Step 2)."""
+        try:
+            db = next(get_db())
+            website_data = self.db_service.get_website_analysis(self.user_id, db)
+            if website_data:
+                logger.info(f"Retrieved website analysis from database for user {self.user_id}")
+            else:
+                logger.warning(f"No website analysis found in database for user {self.user_id}")
+            return website_data
+        except Exception as e:
+            logger.error(f"Error getting website analysis from database: {e}")
             return None
     
     def _get_research_preferences(self) -> Optional[Dict[str, Any]]:
-        """Get research preferences data."""
+        """Get research preferences data from database (Step 3)."""
         try:
             db = next(get_db())
-            research_service = ResearchPreferencesService(db)
-            return research_service.get_research_preferences(self.session_id)
+            research_data = self.db_service.get_research_preferences(self.user_id, db)
+            if research_data:
+                logger.info(f"Retrieved research preferences from database for user {self.user_id}")
+            else:
+                logger.warning(f"No research preferences found in database for user {self.user_id}")
+            return research_data
         except Exception as e:
-            logger.warning(f"Could not get research preferences: {str(e)}")
+            logger.error(f"Error getting research preferences from database: {e}")
             return None
     
     def _get_personalization_settings(self, research_preferences: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
-        """Get personalization settings from research preferences."""
-        if not research_preferences:
+        """Get personalization settings from Step 4 (Persona) database."""
+        try:
+            # Try to get from Step 4 (Persona) in database
+            db = next(get_db())
+            persona_data = self.db_service.get_persona_data(self.user_id, db)
+            
+            if persona_data:
+                logger.info(f"Retrieved persona data from database for user {self.user_id}")
+                # Extract personalization settings from persona data
+                if 'corePersona' in persona_data:
+                    core_persona = persona_data.get('corePersona', {})
+                    return {
+                        'writing_style': core_persona.get('linguistic_fingerprint', {}).get('tone', 'Professional'),
+                        'tone': core_persona.get('tonal_range', {}).get('primary_tone', 'Formal'),
+                        'brand_voice': core_persona.get('identity', {}).get('voice', 'Trustworthy and Expert')
+                    }
+            
+            # Fallback to research preferences if persona data not available
+            if research_preferences:
+                logger.info(f"Using research preferences as fallback for personalization")
+                return {
+                    'writing_style': research_preferences.get('writing_style', {}).get('tone', 'Professional'),
+                    'tone': research_preferences.get('writing_style', {}).get('voice', 'Formal'),
+                    'brand_voice': research_preferences.get('writing_style', {}).get('complexity', 'Trustworthy and Expert')
+                }
+            
+            return None
+        except Exception as e:
+            logger.error(f"Error getting personalization settings from database: {e}")
             return None
-        
-        return {
-            'writing_style': research_preferences.get('writing_style', {}).get('tone', 'Professional'),
-            'tone': research_preferences.get('writing_style', {}).get('voice', 'Formal'),
-            'brand_voice': research_preferences.get('writing_style', {}).get('complexity', 'Trustworthy and Expert')
-        }
     
     def _check_persona_readiness(self, website_analysis: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
         """Check if persona can be generated."""