Recovered state: integrated TrendSurferAgent, restored frontend/backend files, and cleaned up recovery scripts

2026-02-08 13:56:57 +05:30
parent 1db10ccd0f
commit e404a86502
333 changed files with 42223 additions and 10875 deletions
--- a/backend/models/advertools_monitoring_models.py
+++ b/backend/models/advertools_monitoring_models.py
@@ -0,0 +1,100 @@
+"""
+Advertools Monitoring Models
+Database models for tracking Advertools-based SEO intelligence tasks.
+"""
+
+from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean, JSON, Index, ForeignKey
+from sqlalchemy.orm import relationship
+from datetime import datetime
+
+# Import the same Base from enhanced_strategy_models
+from models.enhanced_strategy_models import Base
+
+
+class AdvertoolsTask(Base):
+    """
+    Model for storing Advertools intelligence tasks.
+    Tracks weekly content audits and site health monitoring.
+    """
+    __tablename__ = "advertools_tasks"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    
+    # User and URL Identification
+    user_id = Column(String(255), nullable=False, index=True)
+    website_url = Column(String(500), nullable=False, index=True)
+    
+    # Task Status
+    status = Column(String(50), default='active', index=True)  # 'active', 'failed', 'paused'
+    
+    # Execution Tracking
+    last_executed = Column(DateTime, nullable=True)
+    last_success = Column(DateTime, nullable=True)
+    last_failure = Column(DateTime, nullable=True)
+    failure_reason = Column(Text, nullable=True)
+    
+    # Failure Pattern Tracking
+    consecutive_failures = Column(Integer, default=0)
+    failure_pattern = Column(JSON, nullable=True)
+    
+    # Scheduling
+    next_execution = Column(DateTime, nullable=True, index=True)
+    frequency_days = Column(Integer, default=7)  # Weekly by default
+    
+    # Task Type & Data
+    payload = Column(JSON, nullable=True) # {"type": "content_audit", "website_url": "..."}
+    
+    # Metadata
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+    
+    # Execution Logs Relationship
+    execution_logs = relationship(
+        "AdvertoolsExecutionLog",
+        back_populates="task",
+        cascade="all, delete-orphan"
+    )
+    
+    __table_args__ = (
+        Index('idx_advertools_tasks_user_site', 'user_id', 'website_url'),
+        Index('idx_advertools_tasks_next_execution', 'next_execution'),
+        Index('idx_advertools_tasks_status', 'status'),
+    )
+    
+    def __repr__(self):
+        return f"<AdvertoolsTask(id={self.id}, user_id={self.user_id}, url={self.website_url}, status={self.status})>"
+
+
+class AdvertoolsExecutionLog(Base):
+    """
+    Model for storing Advertools execution logs.
+    """
+    __tablename__ = "advertools_execution_logs"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    
+    # Task Reference
+    task_id = Column(Integer, ForeignKey("advertools_tasks.id"), nullable=False, index=True)
+    
+    # Execution Details
+    execution_date = Column(DateTime, default=datetime.utcnow, nullable=False)
+    status = Column(String(50), nullable=False)  # 'success', 'failed', 'skipped', 'running'
+    
+    # Results
+    result_data = Column(JSON, nullable=True)
+    error_message = Column(Text, nullable=True)
+    execution_time_ms = Column(Integer, nullable=True)
+    
+    # Metadata
+    created_at = Column(DateTime, default=datetime.utcnow)
+    
+    # Relationship to task
+    task = relationship("AdvertoolsTask", back_populates="execution_logs")
+    
+    __table_args__ = (
+        Index('idx_advertools_execution_logs_task_date', 'task_id', 'execution_date'),
+        Index('idx_advertools_execution_logs_status', 'status'),
+    )
+    
+    def __repr__(self):
+        return f"<AdvertoolsExecutionLog(id={self.id}, task_id={self.task_id}, status={self.status}, execution_date={self.execution_date})>"
--- a/backend/models/agent_activity_models.py
+++ b/backend/models/agent_activity_models.py
@@ -0,0 +1,109 @@
+from datetime import datetime
+
+from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean, JSON, ForeignKey, Index, Float
+from sqlalchemy.orm import relationship
+
+from models.enhanced_strategy_models import Base
+
+
+class AgentRun(Base):
+    __tablename__ = "agent_runs"
+
+    id = Column(Integer, primary_key=True, index=True)
+    user_id = Column(String(255), nullable=False, index=True)
+    agent_type = Column(String(100), nullable=False, index=True)
+    prompt = Column(Text, nullable=True)
+    status = Column(String(30), nullable=False, default="running", index=True)
+    success = Column(Boolean, nullable=True)
+    error_message = Column(Text, nullable=True)
+    result_summary = Column(Text, nullable=True)
+    mlflow_run_id = Column(String(255), nullable=True)
+    started_at = Column(DateTime, default=datetime.utcnow, index=True)
+    finished_at = Column(DateTime, nullable=True, index=True)
+
+    events = relationship("AgentEvent", back_populates="run", cascade="all, delete-orphan")
+
+
+class AgentEvent(Base):
+    __tablename__ = "agent_events"
+
+    id = Column(Integer, primary_key=True, index=True)
+    run_id = Column(Integer, ForeignKey("agent_runs.id"), nullable=True, index=True)
+    user_id = Column(String(255), nullable=False, index=True)
+    agent_type = Column(String(100), nullable=True, index=True)
+    event_type = Column(String(50), nullable=False, index=True)
+    severity = Column(String(20), nullable=False, default="info", index=True)
+    message = Column(Text, nullable=True)
+    payload = Column(JSON, nullable=True)
+    created_at = Column(DateTime, default=datetime.utcnow, index=True)
+
+    run = relationship("AgentRun", back_populates="events")
+
+
+class AgentAlert(Base):
+    __tablename__ = "agent_alerts"
+
+    id = Column(Integer, primary_key=True, index=True)
+    user_id = Column(String(255), nullable=False, index=True)
+    source = Column(String(30), nullable=False, default="agents", index=True)
+    alert_type = Column(String(50), nullable=False, index=True)
+    severity = Column(String(20), nullable=False, default="info", index=True)
+    title = Column(String(255), nullable=False)
+    message = Column(Text, nullable=False)
+    cta_path = Column(String(255), nullable=True)
+    payload = Column(JSON, nullable=True)
+    dedupe_key = Column(String(255), nullable=True, index=True)
+    created_at = Column(DateTime, default=datetime.utcnow, index=True)
+    read_at = Column(DateTime, nullable=True, index=True)
+
+
+Index("ix_agent_alerts_user_unread", AgentAlert.user_id, AgentAlert.read_at)
+
+
+class AgentApprovalRequest(Base):
+    __tablename__ = "agent_approval_requests"
+
+    id = Column(Integer, primary_key=True, index=True)
+    user_id = Column(String(255), nullable=False, index=True)
+    run_id = Column(Integer, ForeignKey("agent_runs.id"), nullable=True, index=True)
+    agent_type = Column(String(100), nullable=True, index=True)
+    action_id = Column(String(255), nullable=False, index=True)
+    action_type = Column(String(255), nullable=False, index=True)
+    target_resource = Column(String(255), nullable=True)
+    risk_level = Column(Float, nullable=False, default=0.5)
+    payload = Column(JSON, nullable=True)
+    status = Column(String(30), nullable=False, default="pending", index=True)
+    expires_at = Column(DateTime, nullable=True, index=True)
+    decided_at = Column(DateTime, nullable=True, index=True)
+    decision = Column(String(30), nullable=True)
+    user_comments = Column(Text, nullable=True)
+    created_at = Column(DateTime, default=datetime.utcnow, index=True)
+
+
+Index("ix_agent_approval_user_status", AgentApprovalRequest.user_id, AgentApprovalRequest.status)
+
+
+class AgentProfile(Base):
+    __tablename__ = "agent_profiles"
+
+    id = Column(Integer, primary_key=True, index=True)
+    user_id = Column(String(255), nullable=False, index=True)
+    agent_key = Column(String(100), nullable=False, index=True)
+    agent_type = Column(String(100), nullable=True, index=True)
+
+    display_name = Column(String(255), nullable=True)
+    enabled = Column(Boolean, nullable=False, default=True, index=True)
+
+    schedule = Column(JSON, nullable=True)
+    notification_prefs = Column(JSON, nullable=True)
+
+    tone = Column(JSON, nullable=True)
+    system_prompt = Column(Text, nullable=True)
+    task_prompt_template = Column(Text, nullable=True)
+    reporting_prefs = Column(JSON, nullable=True)
+
+    created_at = Column(DateTime, default=datetime.utcnow, index=True)
+    updated_at = Column(DateTime, default=datetime.utcnow, index=True)
+
+
+Index("ix_agent_profiles_user_key", AgentProfile.user_id, AgentProfile.agent_key, unique=True)
--- a/backend/models/api_monitoring.py
+++ b/backend/models/api_monitoring.py
@@ -30,10 +30,10 @@ class APIRequest(Base):
    
    # Indexes for fast queries
    __table_args__ = (
-        Index('idx_timestamp', 'timestamp'),
-        Index('idx_path_method', 'path', 'method'),
-        Index('idx_status_code', 'status_code'),
-        Index('idx_user_id', 'user_id'),
+        Index('idx_api_req_timestamp', 'timestamp'),
+        Index('idx_api_req_path_method', 'path', 'method'),
+        Index('idx_api_req_status_code', 'status_code'),
+        Index('idx_api_req_user_id', 'user_id'),
    )

 class APIEndpointStats(Base):
@@ -56,9 +56,9 @@ class APIEndpointStats(Base):
    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
    
    __table_args__ = (
-        Index('idx_endpoint', 'endpoint'),
-        Index('idx_total_requests', 'total_requests'),
-        Index('idx_avg_duration', 'avg_duration'),
+        Index('idx_api_stats_endpoint', 'endpoint'),
+        Index('idx_api_stats_total_requests', 'total_requests'),
+        Index('idx_api_stats_avg_duration', 'avg_duration'),
    )

 class SystemHealth(Base):
@@ -78,8 +78,8 @@ class SystemHealth(Base):
    metrics = Column(JSON, nullable=True)  # Additional metrics
    
    __table_args__ = (
-        Index('idx_timestamp', 'timestamp'),
-        Index('idx_status', 'status'),
+        Index('idx_sys_health_timestamp', 'timestamp'),
+        Index('idx_sys_health_status', 'status'),
    )

 class CachePerformance(Base):
@@ -97,6 +97,6 @@ class CachePerformance(Base):
    total_requests = Column(Integer, default=0)
    
    __table_args__ = (
-        Index('idx_timestamp', 'timestamp'),
+        Index('idx_cache_perf_timestamp', 'timestamp'),
        Index('idx_cache_type', 'cache_type'),
    )
--- a/backend/models/business_info_request.py
+++ b/backend/models/business_info_request.py
@@ -4,7 +4,7 @@ from typing import Optional
 from datetime import datetime

 class BusinessInfoRequest(BaseModel):
-    user_id: Optional[int] = None
+    user_id: Optional[str] = None
    business_description: str = Field(..., min_length=10, max_length=1000, description="Description of the business")
    industry: Optional[str] = Field(None, max_length=100, description="Industry sector")
    target_audience: Optional[str] = Field(None, max_length=500, description="Target audience description")
@@ -12,7 +12,7 @@ class BusinessInfoRequest(BaseModel):

 class BusinessInfoResponse(BaseModel):
    id: int
-    user_id: Optional[int]
+    user_id: Optional[str]
    business_description: str
    industry: Optional[str]
    target_audience: Optional[str]
--- a/backend/models/component_logic.py
+++ b/backend/models/component_logic.py
@@ -255,6 +255,8 @@ class StyleDetectionResponse(BaseModel):
    style_analysis: Optional[Dict[str, Any]] = None
    style_patterns: Optional[Dict[str, Any]] = None
    style_guidelines: Optional[Dict[str, Any]] = None
+    seo_audit: Optional[Dict[str, Any]] = None
+    sitemap_analysis: Optional[Dict[str, Any]] = None
    error: Optional[str] = None
    warning: Optional[str] = None
    timestamp: str 
--- a/backend/models/crawled_content.py
+++ b/backend/models/crawled_content.py
@@ -0,0 +1,34 @@
+from sqlalchemy import Column, Integer, String, Text, DateTime, JSON, Index
+from datetime import datetime
+from models.enhanced_strategy_models import Base
+
+class EndUserWebsiteContent(Base):
+    """
+    Model for storing crawled content from the end user's website.
+    """
+    __tablename__ = "end_user_website_content"
+
+    id = Column(Integer, primary_key=True, index=True)
+    user_id = Column(String(255), nullable=False, index=True)
+    website_url = Column(String(500), nullable=False, index=True)
+    
+    # Page details
+    url = Column(String(2048), nullable=False)
+    title = Column(String(1000), nullable=True)
+    content = Column(Text, nullable=True)  # Main content
+    raw_html = Column(Text, nullable=True) # Raw HTML if needed (maybe truncate or store separately)
+    published_date = Column(DateTime, nullable=True)
+    
+    # Metadata
+    metadata_info = Column(JSON, nullable=True) # Any other metadata
+    
+    # Crawl info
+    crawled_at = Column(DateTime, default=datetime.utcnow)
+    status_code = Column(Integer, nullable=True)
+    
+    __table_args__ = (
+        Index('idx_end_user_website_content_user_url', 'user_id', 'url', mysql_length={'url': 255}),
+    )
+
+    def __repr__(self):
+        return f"<EndUserWebsiteContent(id={self.id}, user_id={self.user_id}, url={self.url})>"
--- a/backend/models/daily_workflow_models.py
+++ b/backend/models/daily_workflow_models.py
@@ -0,0 +1,49 @@
+from datetime import datetime
+
+from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean, JSON, ForeignKey, Index
+from sqlalchemy.orm import relationship
+
+from models.enhanced_strategy_models import Base
+
+
+class DailyWorkflowPlan(Base):
+    __tablename__ = "daily_workflow_plans"
+
+    id = Column(Integer, primary_key=True, index=True)
+    user_id = Column(String(255), nullable=False, index=True)
+    date = Column(String(10), nullable=False, index=True)
+    source = Column(String(30), nullable=False, default="agent")
+    plan_json = Column(JSON, nullable=True)
+    generation_run_id = Column(Integer, nullable=True, index=True)
+    created_at = Column(DateTime, default=datetime.utcnow, index=True)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, index=True)
+
+    tasks = relationship("DailyWorkflowTask", back_populates="plan", cascade="all, delete-orphan")
+
+
+class DailyWorkflowTask(Base):
+    __tablename__ = "daily_workflow_tasks"
+
+    id = Column(Integer, primary_key=True, index=True)
+    plan_id = Column(Integer, ForeignKey("daily_workflow_plans.id"), nullable=False, index=True)
+    user_id = Column(String(255), nullable=False, index=True)
+    pillar_id = Column(String(30), nullable=False, index=True)
+    title = Column(String(255), nullable=False)
+    description = Column(Text, nullable=False)
+    status = Column(String(30), nullable=False, default="pending", index=True)
+    priority = Column(String(10), nullable=False, default="medium", index=True)
+    estimated_time = Column(Integer, nullable=False, default=15)
+    action_type = Column(String(20), nullable=False, default="navigate")
+    action_url = Column(String(255), nullable=True)
+    enabled = Column(Boolean, nullable=False, default=True)
+    dependencies = Column(JSON, nullable=True)
+    metadata_json = Column("metadata", JSON, nullable=True)
+    created_at = Column(DateTime, default=datetime.utcnow, index=True)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, index=True)
+    decided_at = Column(DateTime, nullable=True, index=True)
+    completion_notes = Column(Text, nullable=True)
+
+    plan = relationship("DailyWorkflowPlan", back_populates="tasks")
+
+
+Index("ix_daily_workflow_plans_user_date", DailyWorkflowPlan.user_id, DailyWorkflowPlan.date, unique=True)
--- a/backend/models/enhanced_strategy_models.py
+++ b/backend/models/enhanced_strategy_models.py
@@ -17,7 +17,7 @@ class EnhancedContentStrategy(Base):
    
    # Primary fields
    id = Column(Integer, primary_key=True)
-    user_id = Column(Integer, nullable=False)
+    user_id = Column(String(255), nullable=False)
    name = Column(String(255), nullable=False)
    industry = Column(String(100), nullable=True)
    
@@ -186,7 +186,7 @@ class EnhancedAIAnalysisResult(Base):
    __tablename__ = "enhanced_ai_analysis_results"
    
    id = Column(Integer, primary_key=True)
-    user_id = Column(Integer, nullable=False)
+    user_id = Column(String(255), nullable=False)
    strategy_id = Column(Integer, ForeignKey("enhanced_content_strategies.id"), nullable=True)
    
    # Analysis type for the 5 specialized prompts
@@ -244,7 +244,7 @@ class OnboardingDataIntegration(Base):
    __tablename__ = "onboarding_data_integrations"
    
    id = Column(Integer, primary_key=True)
-    user_id = Column(Integer, nullable=False)
+    user_id = Column(String(255), nullable=False)
    strategy_id = Column(Integer, ForeignKey("enhanced_content_strategies.id"), nullable=True)
    
    # Legacy onboarding storage fields (match existing DB schema)
@@ -275,6 +275,7 @@ class OnboardingDataIntegration(Base):
            'website_analysis_data': self.website_analysis_data,
            'research_preferences_data': self.research_preferences_data,
            'api_keys_data': self.api_keys_data,
+            'canonical_profile': self.canonical_profile,
            'field_mappings': self.field_mappings,
            'auto_populated_fields': self.auto_populated_fields,
            'user_overrides': self.user_overrides,
@@ -291,7 +292,7 @@ class ContentStrategyAutofillInsights(Base):

    id = Column(Integer, primary_key=True)
    strategy_id = Column(Integer, ForeignKey("enhanced_content_strategies.id"), nullable=False)
-    user_id = Column(Integer, nullable=False)
+    user_id = Column(String(255), nullable=False)

    # Full snapshot of accepted inputs and transparency at time of strategy creation/confirmation
    accepted_fields = Column(JSON, nullable=False)
@@ -304,4 +305,4 @@ class ContentStrategyAutofillInsights(Base):
    created_at = Column(DateTime, default=datetime.utcnow)

    # Relationship back to strategy
-    strategy = relationship("EnhancedContentStrategy", back_populates="autofill_insights") 
+    strategy = relationship("EnhancedContentStrategy", back_populates="autofill_insights") 
--- a/backend/models/monitoring_models.py
+++ b/backend/models/monitoring_models.py
@@ -65,7 +65,7 @@ class StrategyPerformanceMetrics(Base):
    
    id = Column(Integer, primary_key=True, index=True)
    strategy_id = Column(Integer, ForeignKey("enhanced_content_strategies.id"), nullable=False)
-    user_id = Column(Integer, nullable=False)
+    user_id = Column(String(255), nullable=False)
    metric_date = Column(DateTime, default=datetime.utcnow)
    traffic_growth_percentage = Column(Integer, nullable=True)
    engagement_rate_percentage = Column(Integer, nullable=True)
--- a/backend/models/oauth_token_monitoring_models.py
+++ b/backend/models/oauth_token_monitoring_models.py
@@ -52,11 +52,10 @@ class OAuthTokenMonitoringTask(Base):
        cascade="all, delete-orphan"
    )
    
-    # Indexes for efficient queries
    __table_args__ = (
-        Index('idx_user_platform', 'user_id', 'platform'),
-        Index('idx_next_check', 'next_check'),
-        Index('idx_status', 'status'),
+        Index('idx_oauth_token_tasks_user_platform', 'user_id', 'platform'),
+        Index('idx_oauth_token_tasks_next_check', 'next_check'),
+        Index('idx_oauth_token_tasks_status', 'status'),
    )
    
    def __repr__(self):
@@ -91,10 +90,9 @@ class OAuthTokenExecutionLog(Base):
    # Relationship to task
    task = relationship("OAuthTokenMonitoringTask", back_populates="execution_logs")
    
-    # Indexes for efficient queries
    __table_args__ = (
-        Index('idx_task_execution_date', 'task_id', 'execution_date'),
-        Index('idx_status', 'status'),
+        Index('idx_oauth_token_logs_task_execution_date', 'task_id', 'execution_date'),
+        Index('idx_oauth_token_logs_status', 'status'),
    )
    
    def __repr__(self):
--- a/backend/models/onboarding.py
+++ b/backend/models/onboarding.py
@@ -1,4 +1,4 @@
-from sqlalchemy import Column, Integer, String, Float, DateTime, ForeignKey, func, JSON, Text, Boolean
+from sqlalchemy import Column, Integer, String, Float, DateTime, ForeignKey, func, JSON, Text, Boolean, UniqueConstraint
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import relationship
 import datetime
@@ -61,13 +61,16 @@ class WebsiteAnalysis(Base):
    target_audience = Column(JSON)  # Demographics, expertise level, industry focus
    content_type = Column(JSON)  # Primary type, secondary types, purpose
    recommended_settings = Column(JSON)  # Writing tone, target audience, content type
-    # brand_analysis = Column(JSON)  # Brand voice, values, positioning, competitive differentiation
-    # content_strategy_insights = Column(JSON)  # SWOT analysis, strengths, weaknesses, opportunities, threats
+    brand_analysis = Column(JSON)  # Brand voice, values, positioning, competitive differentiation
+    content_strategy_insights = Column(JSON)  # SWOT analysis, strengths, weaknesses, opportunities, threats
+    social_media_presence = Column(JSON)  # Social media accounts and metrics
    
    # Crawl results
    crawl_result = Column(JSON)  # Raw crawl data
    style_patterns = Column(JSON)  # Writing patterns analysis
    style_guidelines = Column(JSON)  # Generated guidelines
+    seo_audit = Column(JSON)  # Comprehensive SEO audit results
+    strategic_insights_history = Column(JSON)  # Weekly strategic intelligence reports history
    
    # Metadata
    status = Column(String(50), default='completed')  # completed, failed, in_progress
@@ -86,6 +89,7 @@ class WebsiteAnalysis(Base):
        """Convert to dictionary for API responses."""
        return {
            'id': self.id,
+            'session_id': self.session_id,
            'website_url': self.website_url,
            'analysis_date': self.analysis_date.isoformat() if self.analysis_date else None,
            'writing_style': self.writing_style,
@@ -93,11 +97,14 @@ class WebsiteAnalysis(Base):
            'target_audience': self.target_audience,
            'content_type': self.content_type,
            'recommended_settings': self.recommended_settings,
-            # 'brand_analysis': self.brand_analysis,
-            # 'content_strategy_insights': self.content_strategy_insights,
+            'brand_analysis': self.brand_analysis,
+            'content_strategy_insights': self.content_strategy_insights,
+            'social_media_presence': self.social_media_presence,
            'crawl_result': self.crawl_result,
            'style_patterns': self.style_patterns,
            'style_guidelines': self.style_guidelines,
+            'seo_audit': self.seo_audit,
+            'strategic_insights_history': self.strategic_insights_history,
            'status': self.status,
            'error_message': self.error_message,
            'warning_message': self.warning_message,
@@ -105,6 +112,50 @@ class WebsiteAnalysis(Base):
            'updated_at': self.updated_at.isoformat() if self.updated_at else None
        } 

+class SEOPageAudit(Base):
+    __tablename__ = 'seo_page_audits'
+    __table_args__ = (
+        UniqueConstraint('user_id', 'page_url', name='uq_seo_page_audits_user_page'),
+    )
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    user_id = Column(String(255), nullable=False, index=True)
+    website_url = Column(String(500), nullable=False, index=True)
+    page_url = Column(String(1000), nullable=False, index=True)
+
+    overall_score = Column(Integer, nullable=True)
+    status = Column(String(50), default='needs_review', index=True)
+
+    category_scores = Column(JSON)
+    issues = Column(JSON)
+    warnings = Column(JSON)
+    recommendations = Column(JSON)
+    audit_data = Column(JSON)
+
+    analysis_source = Column(String(50), default='onboarding_full_site')
+    last_analyzed_at = Column(DateTime, default=func.now())
+    created_at = Column(DateTime, default=func.now())
+    updated_at = Column(DateTime, default=func.now(), onupdate=func.now())
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'user_id': self.user_id,
+            'website_url': self.website_url,
+            'page_url': self.page_url,
+            'overall_score': self.overall_score,
+            'status': self.status,
+            'category_scores': self.category_scores,
+            'issues': self.issues,
+            'warnings': self.warnings,
+            'recommendations': self.recommendations,
+            'audit_data': self.audit_data,
+            'analysis_source': self.analysis_source,
+            'last_analyzed_at': self.last_analyzed_at.isoformat() if self.last_analyzed_at else None,
+            'created_at': self.created_at.isoformat() if self.created_at else None,
+            'updated_at': self.updated_at.isoformat() if self.updated_at else None,
+        }
+
 class ResearchPreferences(Base):
    """Stores research preferences from onboarding step 3."""
    __tablename__ = 'research_preferences'
@@ -197,7 +248,7 @@ class CompetitorAnalysis(Base):
    
    id = Column(Integer, primary_key=True, autoincrement=True)
    session_id = Column(Integer, ForeignKey('onboarding_sessions.id', ondelete='CASCADE'), nullable=False)
-    competitor_url = Column(String(500), nullable=False)
+    competitor_url = Column(Text, nullable=False)
    competitor_domain = Column(String(255), nullable=True)  # Extracted domain for easier queries
    analysis_date = Column(DateTime, default=func.now())
    
@@ -231,4 +282,4 @@ class CompetitorAnalysis(Base):
            'warning_message': self.warning_message,
            'created_at': self.created_at.isoformat() if self.created_at else None,
            'updated_at': self.updated_at.isoformat() if self.updated_at else None
-        }
+        }
--- a/backend/models/podcast_models.py
+++ b/backend/models/podcast_models.py
@@ -62,7 +62,7 @@ class PodcastProject(Base):
    
    # Composite indexes for common query patterns
    __table_args__ = (
-        Index('idx_user_status_created', 'user_id', 'status', 'created_at'),
-        Index('idx_user_favorite_updated', 'user_id', 'is_favorite', 'updated_at'),
+        Index('idx_podcast_user_status_created', 'user_id', 'status', 'created_at'),
+        Index('idx_podcast_user_favorite_updated', 'user_id', 'is_favorite', 'updated_at'),
    )

--- a/backend/models/product_marketing_models.py
+++ b/backend/models/product_marketing_models.py
@@ -67,8 +67,8 @@ class Campaign(Base):
    
    # Composite indexes
    __table_args__ = (
-        Index('idx_user_status', 'user_id', 'status'),
-        Index('idx_user_created', 'user_id', 'created_at'),
+        Index('idx_pm_campaign_user_status', 'user_id', 'status'),
+        Index('idx_pm_campaign_user_created', 'user_id', 'created_at'),
    )


@@ -109,10 +109,10 @@ class CampaignProposal(Base):
    campaign = relationship("Campaign", back_populates="proposals")
    generated_asset = relationship("CampaignAsset", back_populates="proposal", uselist=False)
    
-    # Composite indexes
+    ## Composite indexes
    __table_args__ = (
-        Index('idx_campaign_node', 'campaign_id', 'asset_node_id'),
-        Index('idx_user_status', 'user_id', 'status'),
+        Index('idx_pm_proposal_campaign_node', 'campaign_id', 'asset_node_id'),
+        Index('idx_pm_proposal_user_status', 'user_id', 'status'),
    )


@@ -156,7 +156,7 @@ class CampaignAsset(Base):
    
    # Composite indexes
    __table_args__ = (
-        Index('idx_campaign_node', 'campaign_id', 'asset_node_id'),
-        Index('idx_user_status', 'user_id', 'status'),
+        Index('idx_pm_asset_campaign_node', 'campaign_id', 'asset_node_id'),
+        Index('idx_pm_asset_user_status', 'user_id', 'status'),
    )

--- a/backend/models/research_models.py
+++ b/backend/models/research_models.py
@@ -53,6 +53,6 @@ class ResearchProject(Base):
    
    # Composite indexes for common query patterns
    __table_args__ = (
-        Index('idx_user_status_created', 'user_id', 'status', 'created_at'),
-        Index('idx_user_favorite_updated', 'user_id', 'is_favorite', 'updated_at'),
+        Index('idx_research_user_status_created', 'user_id', 'status', 'created_at'),
+        Index('idx_research_user_favorite_updated', 'user_id', 'is_favorite', 'updated_at'),
    )
--- a/backend/models/user_business_info.py
+++ b/backend/models/user_business_info.py
@@ -12,7 +12,7 @@ class UserBusinessInfo(Base):
    __tablename__ = 'user_business_info'

    id = Column(Integer, primary_key=True, index=True)
-    user_id = Column(Integer, index=True, nullable=True)
+    user_id = Column(String(255), index=True, nullable=True)
    business_description = Column(Text, nullable=False)
    industry = Column(String(100), nullable=True)
    target_audience = Column(Text, nullable=True)
--- a/backend/models/website_analysis_monitoring_models.py
+++ b/backend/models/website_analysis_monitoring_models.py
@@ -107,3 +107,344 @@ class WebsiteAnalysisExecutionLog(Base):
    def __repr__(self):
        return f"<WebsiteAnalysisExecutionLog(id={self.id}, task_id={self.task_id}, status={self.status}, execution_date={self.execution_date})>"

+
+class OnboardingFullWebsiteAnalysisTask(Base):
+    __tablename__ = "onboarding_full_website_analysis_tasks"
+
+    id = Column(Integer, primary_key=True, index=True)
+
+    user_id = Column(String(255), nullable=False, index=True)
+    website_url = Column(String(500), nullable=False, index=True)
+
+    status = Column(String(50), default='active', index=True)
+
+    last_executed = Column(DateTime, nullable=True)
+    last_success = Column(DateTime, nullable=True)
+    last_failure = Column(DateTime, nullable=True)
+    failure_reason = Column(Text, nullable=True)
+
+    consecutive_failures = Column(Integer, default=0)
+    failure_pattern = Column(JSON, nullable=True)
+
+    next_execution = Column(DateTime, nullable=True, index=True)
+
+    payload = Column(JSON, nullable=True)
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+    execution_logs = relationship(
+        "OnboardingFullWebsiteAnalysisExecutionLog",
+        back_populates="task",
+        cascade="all, delete-orphan"
+    )
+
+    __table_args__ = (
+        Index('idx_onboarding_full_website_analysis_tasks_user_site', 'user_id', 'website_url'),
+        Index('idx_onboarding_full_website_analysis_tasks_next_execution', 'next_execution'),
+        Index('idx_onboarding_full_website_analysis_tasks_status', 'status'),
+    )
+
+    def __repr__(self):
+        return f"<OnboardingFullWebsiteAnalysisTask(id={self.id}, user_id={self.user_id}, url={self.website_url}, status={self.status})>"
+
+
+class OnboardingFullWebsiteAnalysisExecutionLog(Base):
+    __tablename__ = "onboarding_full_website_analysis_execution_logs"
+
+    id = Column(Integer, primary_key=True, index=True)
+
+    task_id = Column(Integer, ForeignKey("onboarding_full_website_analysis_tasks.id"), nullable=False, index=True)
+
+    execution_date = Column(DateTime, default=datetime.utcnow, nullable=False)
+    status = Column(String(50), nullable=False)
+
+    result_data = Column(JSON, nullable=True)
+    error_message = Column(Text, nullable=True)
+    execution_time_ms = Column(Integer, nullable=True)
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+
+    task = relationship("OnboardingFullWebsiteAnalysisTask", back_populates="execution_logs")
+
+    __table_args__ = (
+        Index('idx_onboarding_full_website_analysis_execution_logs_task_date', 'task_id', 'execution_date'),
+        Index('idx_onboarding_full_website_analysis_execution_logs_status', 'status'),
+    )
+
+    def __repr__(self):
+        return f"<OnboardingFullWebsiteAnalysisExecutionLog(id={self.id}, task_id={self.task_id}, status={self.status}, execution_date={self.execution_date})>"
+
+
+class DeepCompetitorAnalysisTask(Base):
+    __tablename__ = "deep_competitor_analysis_tasks"
+
+    id = Column(Integer, primary_key=True, index=True)
+
+    user_id = Column(String(255), nullable=False, index=True)
+    website_url = Column(String(500), nullable=False, index=True)
+
+    status = Column(String(50), default='active', index=True)
+
+    last_executed = Column(DateTime, nullable=True)
+    last_success = Column(DateTime, nullable=True)
+    last_failure = Column(DateTime, nullable=True)
+    failure_reason = Column(Text, nullable=True)
+
+    consecutive_failures = Column(Integer, default=0)
+    failure_pattern = Column(JSON, nullable=True)
+
+    next_execution = Column(DateTime, nullable=True, index=True)
+
+    payload = Column(JSON, nullable=True)
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+    execution_logs = relationship(
+        "DeepCompetitorAnalysisExecutionLog",
+        back_populates="task",
+        cascade="all, delete-orphan"
+    )
+
+    __table_args__ = (
+        Index('idx_deep_competitor_analysis_tasks_user_site', 'user_id', 'website_url'),
+        Index('idx_deep_competitor_analysis_tasks_next_execution', 'next_execution'),
+        Index('idx_deep_competitor_analysis_tasks_status', 'status'),
+    )
+
+    def __repr__(self):
+        return f"<DeepCompetitorAnalysisTask(id={self.id}, user_id={self.user_id}, url={self.website_url}, status={self.status})>"
+
+
+class DeepCompetitorAnalysisExecutionLog(Base):
+    __tablename__ = "deep_competitor_analysis_execution_logs"
+
+    id = Column(Integer, primary_key=True, index=True)
+
+    task_id = Column(Integer, ForeignKey("deep_competitor_analysis_tasks.id"), nullable=False, index=True)
+
+    execution_date = Column(DateTime, default=datetime.utcnow, nullable=False)
+    status = Column(String(50), nullable=False)
+
+    result_data = Column(JSON, nullable=True)
+    error_message = Column(Text, nullable=True)
+    execution_time_ms = Column(Integer, nullable=True)
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+
+    task = relationship("DeepCompetitorAnalysisTask", back_populates="execution_logs")
+
+    __table_args__ = (
+        Index('idx_deep_competitor_analysis_execution_logs_task_date', 'task_id', 'execution_date'),
+        Index('idx_deep_competitor_analysis_execution_logs_status', 'status'),
+    )
+
+    def __repr__(self):
+        return f"<DeepCompetitorAnalysisExecutionLog(id={self.id}, task_id={self.task_id}, status={self.status}, execution_date={self.execution_date})>"
+
+
+class DeepWebsiteCrawlTask(Base):
+    __tablename__ = "deep_website_crawl_tasks"
+
+    id = Column(Integer, primary_key=True, index=True)
+
+    user_id = Column(String(255), nullable=False, index=True)
+    website_url = Column(String(500), nullable=False, index=True)
+
+    status = Column(String(50), default='active', index=True)
+
+    last_executed = Column(DateTime, nullable=True)
+    last_success = Column(DateTime, nullable=True)
+    last_failure = Column(DateTime, nullable=True)
+    failure_reason = Column(Text, nullable=True)
+
+    consecutive_failures = Column(Integer, default=0)
+    failure_pattern = Column(JSON, nullable=True)
+
+    next_execution = Column(DateTime, nullable=True, index=True)
+
+    payload = Column(JSON, nullable=True)
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+    execution_logs = relationship(
+        "DeepWebsiteCrawlExecutionLog",
+        back_populates="task",
+        cascade="all, delete-orphan"
+    )
+
+    __table_args__ = (
+        Index('idx_deep_website_crawl_tasks_user_site', 'user_id', 'website_url'),
+        Index('idx_deep_website_crawl_tasks_next_execution', 'next_execution'),
+        Index('idx_deep_website_crawl_tasks_status', 'status'),
+    )
+
+    def __repr__(self):
+        return f"<DeepWebsiteCrawlTask(id={self.id}, user_id={self.user_id}, url={self.website_url}, status={self.status})>"
+
+
+class DeepWebsiteCrawlExecutionLog(Base):
+    __tablename__ = "deep_website_crawl_execution_logs"
+
+    id = Column(Integer, primary_key=True, index=True)
+
+    task_id = Column(Integer, ForeignKey("deep_website_crawl_tasks.id"), nullable=False, index=True)
+
+    execution_date = Column(DateTime, default=datetime.utcnow, nullable=False)
+    status = Column(String(50), nullable=False)
+
+    result_data = Column(JSON, nullable=True)
+    error_message = Column(Text, nullable=True)
+    execution_time_ms = Column(Integer, nullable=True)
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+
+    task = relationship("DeepWebsiteCrawlTask", back_populates="execution_logs")
+
+    __table_args__ = (
+        Index('idx_deep_website_crawl_execution_logs_task_date', 'task_id', 'execution_date'),
+        Index('idx_deep_website_crawl_execution_logs_status', 'status'),
+    )
+
+    def __repr__(self):
+        return f"<DeepWebsiteCrawlExecutionLog(id={self.id}, task_id={self.task_id}, status={self.status}, execution_date={self.execution_date})>"
+
+
+class SIFIndexingTask(Base):
+    __tablename__ = "sif_indexing_tasks"
+
+    id = Column(Integer, primary_key=True, index=True)
+
+    user_id = Column(String(255), nullable=False, index=True)
+    website_url = Column(String(500), nullable=False, index=True)
+
+    status = Column(String(50), default='active', index=True)
+
+    last_executed = Column(DateTime, nullable=True)
+    last_success = Column(DateTime, nullable=True)
+    last_failure = Column(DateTime, nullable=True)
+    failure_reason = Column(Text, nullable=True)
+
+    consecutive_failures = Column(Integer, default=0)
+    failure_pattern = Column(JSON, nullable=True)
+
+    next_execution = Column(DateTime, nullable=True, index=True)
+    frequency_hours = Column(Integer, default=48)  # Default 48 hours
+
+    payload = Column(JSON, nullable=True)
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+    execution_logs = relationship(
+        "SIFIndexingExecutionLog",
+        back_populates="task",
+        cascade="all, delete-orphan"
+    )
+
+    __table_args__ = (
+        Index('idx_sif_indexing_tasks_user_site', 'user_id', 'website_url'),
+        Index('idx_sif_indexing_tasks_next_execution', 'next_execution'),
+        Index('idx_sif_indexing_tasks_status', 'status'),
+    )
+
+    def __repr__(self):
+        return f"<SIFIndexingTask(id={self.id}, user_id={self.user_id}, url={self.website_url}, status={self.status})>"
+
+
+class SIFIndexingExecutionLog(Base):
+    __tablename__ = "sif_indexing_execution_logs"
+
+    id = Column(Integer, primary_key=True, index=True)
+
+    task_id = Column(Integer, ForeignKey("sif_indexing_tasks.id"), nullable=False, index=True)
+
+    execution_date = Column(DateTime, default=datetime.utcnow, nullable=False)
+    status = Column(String(50), nullable=False)
+
+    result_data = Column(JSON, nullable=True)
+    error_message = Column(Text, nullable=True)
+    execution_time_ms = Column(Integer, nullable=True)
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+
+    task = relationship("SIFIndexingTask", back_populates="execution_logs")
+
+    __table_args__ = (
+        Index('idx_sif_indexing_execution_logs_task_date', 'task_id', 'execution_date'),
+        Index('idx_sif_indexing_execution_logs_status', 'status'),
+    )
+
+    def __repr__(self):
+        return f"<SIFIndexingExecutionLog(id={self.id}, task_id={self.task_id}, status={self.status}, execution_date={self.execution_date})>"
+
+
+class MarketTrendsTask(Base):
+    __tablename__ = "market_trends_tasks"
+
+    id = Column(Integer, primary_key=True, index=True)
+
+    user_id = Column(String(255), nullable=False, index=True)
+    website_url = Column(String(500), nullable=False, index=True)
+
+    status = Column(String(50), default="active", index=True)
+
+    last_executed = Column(DateTime, nullable=True)
+    last_success = Column(DateTime, nullable=True)
+    last_failure = Column(DateTime, nullable=True)
+    failure_reason = Column(Text, nullable=True)
+
+    consecutive_failures = Column(Integer, default=0)
+    failure_pattern = Column(JSON, nullable=True)
+
+    next_execution = Column(DateTime, nullable=True, index=True)
+    frequency_hours = Column(Integer, default=72)
+
+    payload = Column(JSON, nullable=True)
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+    execution_logs = relationship(
+        "MarketTrendsExecutionLog",
+        back_populates="task",
+        cascade="all, delete-orphan",
+    )
+
+    __table_args__ = (
+        Index("idx_market_trends_tasks_user_site", "user_id", "website_url"),
+        Index("idx_market_trends_tasks_next_execution", "next_execution"),
+        Index("idx_market_trends_tasks_status", "status"),
+    )
+
+    def __repr__(self):
+        return f"<MarketTrendsTask(id={self.id}, user_id={self.user_id}, url={self.website_url}, status={self.status})>"
+
+
+class MarketTrendsExecutionLog(Base):
+    __tablename__ = "market_trends_execution_logs"
+
+    id = Column(Integer, primary_key=True, index=True)
+
+    task_id = Column(Integer, ForeignKey("market_trends_tasks.id"), nullable=False, index=True)
+
+    execution_date = Column(DateTime, default=datetime.utcnow, nullable=False)
+    status = Column(String(50), nullable=False)
+
+    result_data = Column(JSON, nullable=True)
+    error_message = Column(Text, nullable=True)
+    execution_time_ms = Column(Integer, nullable=True)
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+
+    task = relationship("MarketTrendsTask", back_populates="execution_logs")
+
+    __table_args__ = (
+        Index("idx_market_trends_execution_logs_task_date", "task_id", "execution_date"),
+        Index("idx_market_trends_execution_logs_status", "status"),
+    )
+
+    def __repr__(self):
+        return f"<MarketTrendsExecutionLog(id={self.id}, task_id={self.task_id}, status={self.status}, execution_date={self.execution_date})>"