Recovered state: integrated TrendSurferAgent, restored frontend/backend files, and cleaned up recovery scripts

This commit is contained in:
ajaysi
2026-02-08 13:56:57 +05:30
parent 1db10ccd0f
commit e404a86502
333 changed files with 42223 additions and 10875 deletions

View File

@@ -1,37 +1,54 @@
"""
Database Migration Script for Billing System
Creates all tables needed for billing, usage tracking, and subscription management.
Supports multi-tenant architecture.
"""
import sys
import os
import argparse
from pathlib import Path
# Add the backend directory to Python path
backend_dir = Path(__file__).parent.parent
sys.path.insert(0, str(backend_dir))
from sqlalchemy import create_engine, text
from sqlalchemy import create_engine, text, inspect
from sqlalchemy.orm import sessionmaker
from loguru import logger
import traceback
# Import models
from models.subscription_models import Base as SubscriptionBase
from services.database import DATABASE_URL
from services.database import get_engine_for_user, get_all_user_ids, init_user_database
from services.subscription.pricing_service import PricingService
def create_billing_tables():
"""Create all billing and subscription-related tables."""
def check_existing_tables(engine):
"""Check if billing tables exist."""
if engine is None:
return False
try:
inspector = inspect(engine)
tables = inspector.get_table_names()
# Check for a key table
return 'subscription_plans' in tables
except Exception as e:
logger.warning(f"Error checking existing tables: {e}")
return False
def create_billing_tables(user_id):
"""Create all billing and subscription-related tables for a specific user."""
try:
# Create engine
engine = create_engine(DATABASE_URL, echo=False)
logger.info(f"Setting up billing tables for user: {user_id}")
# Create all tables
# Get engine for user
engine = get_engine_for_user(user_id)
# Create all tables (idempotent)
logger.debug("Creating billing and subscription system tables...")
SubscriptionBase.metadata.create_all(bind=engine)
logger.debug("✅ Billing and subscription tables created successfully")
logger.debug("✅ Billing and subscription tables created/verified")
# Create session for data initialization
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
@@ -49,6 +66,8 @@ def create_billing_tables():
pricing_service.initialize_default_plans()
logger.debug("✅ Default subscription plans initialized")
db.commit()
except Exception as e:
logger.error(f"Error initializing default data: {e}")
logger.error(traceback.format_exc())
@@ -57,15 +76,17 @@ def create_billing_tables():
finally:
db.close()
logger.info("✅ Billing system setup completed successfully!")
logger.info(f"✅ Billing system setup completed successfully for {user_id}!")
# Display summary
display_setup_summary(engine)
return True
except Exception as e:
logger.error(f"❌ Error creating billing tables: {e}")
logger.error(f"❌ Error creating billing tables for {user_id}: {e}")
logger.error(traceback.format_exc())
raise
return False
def display_setup_summary(engine):
"""Display a summary of the created tables and data."""
@@ -144,74 +165,36 @@ def display_setup_summary(engine):
logger.warning(f"Could not check API pricing: {e}")
logger.info("\n" + "="*60)
logger.info("NEXT STEPS:")
logger.info("="*60)
logger.info("1. Billing system is ready for use")
logger.info("2. API endpoints are available at:")
logger.info(" GET /api/subscription/plans")
logger.info(" GET /api/subscription/usage/{user_id}")
logger.info(" GET /api/subscription/dashboard/{user_id}")
logger.info(" GET /api/subscription/pricing")
logger.info("\n3. Frontend billing dashboard is integrated")
logger.info("4. Usage tracking middleware is active")
logger.info("5. Real-time cost monitoring is enabled")
logger.info("="*60)
except Exception as e:
logger.error(f"Error displaying summary: {e}")
def check_existing_tables(engine):
"""Check if billing tables already exist."""
try:
with engine.connect() as conn:
# Check for billing tables
check_query = text("""
SELECT name FROM sqlite_master
WHERE type='table' AND (
name = 'subscription_plans' OR
name = 'user_subscriptions' OR
name = 'api_usage_logs' OR
name = 'usage_summaries' OR
name = 'api_provider_pricing' OR
name = 'usage_alerts'
)
""")
result = conn.execute(check_query)
existing_tables = result.fetchall()
if existing_tables:
logger.warning(f"Found existing billing tables: {[t[0] for t in existing_tables]}")
logger.debug("Tables already exist. Skipping creation to preserve data.")
return False
return True
except Exception as e:
logger.error(f"Error checking existing tables: {e}")
return True # Proceed anyway
if __name__ == "__main__":
logger.debug("🚀 Starting billing system database migration...")
parser = argparse.ArgumentParser(description='Create billing tables for a user.')
parser.add_argument('--user_id', type=str, help='Specific user ID to setup billing for')
parser.add_argument('--all', action='store_true', help='Setup billing for ALL users')
try:
# Create engine to check existing tables
engine = create_engine(DATABASE_URL, echo=False)
args = parser.parse_args()
if args.user_id:
create_billing_tables(args.user_id)
elif args.all:
user_ids = get_all_user_ids()
logger.info(f"Found {len(user_ids)} users to process")
for uid in user_ids:
create_billing_tables(uid)
else:
logger.warning("No user_id provided. Using default behavior (checking for single tenant or exiting).")
logger.warning("Usage: python create_billing_tables.py --user_id <user_id> OR --all")
# Check existing tables
if not check_existing_tables(engine):
logger.debug("✅ Billing tables already exist, skipping creation")
sys.exit(0)
# Create tables and initialize data
create_billing_tables()
logger.info("✅ Billing system migration completed successfully!")
except KeyboardInterrupt:
logger.warning("Migration cancelled by user")
sys.exit(0)
except Exception as e:
logger.error(f"❌ Migration failed: {e}")
sys.exit(1)
# Fallback: if there's only one user, maybe we can guess?
# But safer to just exit or ask for input.
# For now, let's try to discover users and if only 1, do it.
user_ids = get_all_user_ids()
if len(user_ids) == 1:
logger.info(f"Single user found: {user_ids[0]}. Proceeding...")
create_billing_tables(user_ids[0])
elif len(user_ids) > 1:
logger.error(f"Multiple users found {user_ids}. Please specify --user_id or --all")
else:
logger.error("No users found.")

View File

@@ -1,3 +1,4 @@
#!/usr/bin/env python3
"""
Database migration script to create comprehensive user data cache table.
Run this script to add the cache table to your database.
@@ -10,13 +11,20 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
from loguru import logger
import os
import argparse
from services.database import get_user_db_path
def create_cache_table():
def create_cache_table(user_id=None):
"""Create the comprehensive user data cache table."""
try:
# Get database URL from environment or use default
database_url = os.getenv('DATABASE_URL', 'sqlite:///alwrity.db')
if user_id:
db_path = get_user_db_path(user_id)
database_url = f'sqlite:///{db_path}'
logger.info(f"Targeting user database: {db_path}")
else:
logger.error("❌ Error: user_id is required to create cache table.")
return False
# Create engine
engine = create_engine(database_url)
@@ -87,11 +95,17 @@ def create_cache_table():
db.close()
return False
def drop_cache_table():
def drop_cache_table(user_id=None):
"""Drop the comprehensive user data cache table (for testing)."""
try:
# Get database URL from environment or use default
database_url = os.getenv('DATABASE_URL', 'sqlite:///alwrity.db')
if user_id:
db_path = get_user_db_path(user_id)
database_url = f'sqlite:///{db_path}'
logger.info(f"Targeting user database: {db_path}")
else:
logger.error("❌ Error: user_id is required to drop cache table.")
return False
# Create engine
engine = create_engine(database_url)
@@ -100,15 +114,14 @@ def drop_cache_table():
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
db = SessionLocal()
# Drop table
logger.info("Dropping comprehensive_user_data_cache table...")
db.execute(text("DROP TABLE IF EXISTS comprehensive_user_data_cache;"))
db.execute(text("DROP TABLE IF EXISTS comprehensive_user_data_cache"))
db.commit()
logger.info("✅ Table dropped successfully")
logger.info("✅ Comprehensive user data cache table dropped successfully!")
db.close()
return True
except Exception as e:
logger.error(f"❌ Error dropping cache table: {str(e)}")
if 'db' in locals():
@@ -116,25 +129,21 @@ def drop_cache_table():
return False
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description="Manage comprehensive user data cache table")
parser.add_argument("--action", choices=["create", "drop"], default="create",
help="Action to perform (create or drop table)")
parser = argparse.ArgumentParser(description="Create comprehensive user data cache table")
parser.add_argument("--user_id", help="Target specific user ID")
parser.add_argument("--drop", action="store_true", help="Drop the table instead of creating it")
args = parser.parse_args()
if args.action == "create":
success = create_cache_table()
if success:
logger.info("🎉 Cache table setup completed successfully!")
else:
logger.error("💥 Cache table setup failed!")
sys.exit(1)
elif args.action == "drop":
success = drop_cache_table()
if success:
logger.info("🗑️ Cache table dropped successfully!")
else:
logger.error("💥 Failed to drop cache table!")
sys.exit(1)
if args.drop:
logger.info("🗑️ Dropping comprehensive user data cache table...")
success = drop_cache_table(args.user_id)
else:
logger.info("🚀 Creating comprehensive user data cache table...")
success = create_cache_table(args.user_id)
if success:
logger.success("🎉 Operation completed successfully!")
sys.exit(0)
else:
logger.error("❌ Operation failed!")
sys.exit(1)

View File

@@ -11,12 +11,20 @@ from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
from loguru import logger
import os
import argparse
from services.database import get_user_db_path
def create_monitoring_tables():
def create_monitoring_tables(user_id=None):
"""Create the API monitoring tables."""
try:
# Get database URL from environment or use default
database_url = os.getenv('DATABASE_URL', 'sqlite:///alwrity.db')
# Get database URL
if user_id:
db_path = get_user_db_path(user_id)
database_url = f'sqlite:///{db_path}'
logger.info(f"Targeting user database: {db_path}")
else:
logger.error("❌ Error: user_id is required to create monitoring tables.")
return False
# Create engine
engine = create_engine(database_url)
@@ -138,11 +146,17 @@ def create_monitoring_tables():
db.close()
return False
def drop_monitoring_tables():
def drop_monitoring_tables(user_id=None):
"""Drop the API monitoring tables (for testing)."""
try:
# Get database URL from environment or use default
database_url = os.getenv('DATABASE_URL', 'sqlite:///alwrity.db')
# Get database URL
if user_id:
db_path = get_user_db_path(user_id)
database_url = f'sqlite:///{db_path}'
logger.info(f"Targeting user database: {db_path}")
else:
logger.error("❌ Error: user_id is required to drop monitoring tables.")
return False
# Create engine
engine = create_engine(database_url)
@@ -176,18 +190,19 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Manage API monitoring tables")
parser.add_argument("--action", choices=["create", "drop"], default="create",
help="Action to perform (create or drop tables)")
parser.add_argument("--user_id", help="Target specific user ID")
args = parser.parse_args()
if args.action == "create":
success = create_monitoring_tables()
success = create_monitoring_tables(args.user_id)
if success:
logger.info("🎉 API monitoring tables setup completed successfully!")
else:
logger.error("💥 API monitoring tables setup failed!")
sys.exit(1)
elif args.action == "drop":
success = drop_monitoring_tables()
success = drop_monitoring_tables(args.user_id)
if success:
logger.info("🗑️ API monitoring tables dropped successfully!")
else:

View File

@@ -7,6 +7,7 @@ Drops old conflicting indexes and ensures proper index names.
import sys
import os
import sqlite3
import argparse
from pathlib import Path
from loguru import logger
@@ -14,9 +15,17 @@ from loguru import logger
backend_dir = Path(__file__).parent.parent
sys.path.insert(0, str(backend_dir))
def fix_indexes():
from services.database import get_user_db_path
def fix_indexes(user_id=None):
"""Fix index name conflicts."""
db_path = backend_dir / "alwrity.db"
if user_id:
db_path = Path(get_user_db_path(user_id))
logger.info(f"Targeting user database: {db_path}")
else:
# Legacy fallback
db_path = Path(get_user_db_path('alwrity'))
logger.info(f"Targeting default/legacy database: {db_path}")
if not db_path.exists():
logger.error(f"Database not found at {db_path}")
@@ -79,8 +88,12 @@ def fix_indexes():
conn.close()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Fix website analysis index conflicts.")
parser.add_argument("--user_id", help="Target specific user ID")
args = parser.parse_args()
logger.info("🔧 Fixing website analysis index conflicts...")
success = fix_indexes()
success = fix_indexes(args.user_id)
if success:
logger.info("✅ Index fix complete. You can now restart the backend.")
sys.exit(0)

View File

@@ -17,6 +17,10 @@ from models.enhanced_calendar_models import (
ContentTrendAnalysis, ContentOptimization, CalendarGenerationSession,
Base as EnhancedCalendarBase
)
from models.enhanced_strategy_models import (
EnhancedContentStrategy, EnhancedAIAnalysisResult, OnboardingDataIntegration,
Base as EnhancedStrategyBase
)
def migrate_table(db, table_name, base_metadata):
"""Migrate user_id column for a specific table from INTEGER to VARCHAR(255)."""
@@ -27,13 +31,7 @@ def migrate_table(db, table_name, base_metadata):
check_table_query = f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}';"
result = db.execute(text(check_table_query))
if not result.scalar():
logger.warning(f"Table '{table_name}' does not exist. Skipping check, but will try to create it.")
# If it doesn't exist, we can just create it with the new schema
try:
base_metadata.create_all(bind=engine, tables=[base_metadata.tables[table_name]], checkfirst=True)
logger.success(f"✅ Created {table_name} with new schema")
except Exception as e:
logger.error(f"Failed to create {table_name}: {e}")
logger.warning(f"Table '{table_name}' does not exist. Skipping migration for this table.")
return True
# Check current column type
@@ -131,6 +129,16 @@ def migrate_all():
for table in ec_tables:
migrate_table(db, table, EnhancedCalendarBase.metadata)
# Enhanced Strategy Tables
es_tables = [
"enhanced_content_strategies",
"enhanced_ai_analysis_results",
"onboarding_data_integrations"
]
for table in es_tables:
migrate_table(db, table, EnhancedStrategyBase.metadata)
finally:
db.close()

View File

@@ -14,11 +14,18 @@ from loguru import logger
backend_dir = Path(__file__).parent.parent
sys.path.insert(0, str(backend_dir))
def run_migration():
from services.database import get_user_db_path
def run_migration(user_id=None):
"""Run the business info table migration."""
try:
# Get the database path
db_path = backend_dir / "alwrity.db"
if user_id:
db_path = Path(get_user_db_path(user_id))
logger.info(f"Targeting user database: {db_path}")
else:
logger.error("❌ Error: user_id is required for migration.")
return False
logger.info(f"🔄 Starting business info table migration...")
logger.info(f"📁 Database path: {db_path}")
@@ -90,7 +97,11 @@ def run_migration():
if __name__ == "__main__":
logger.info("🚀 Starting ALwrity Business Info Migration")
success = run_migration()
parser = argparse.ArgumentParser(description="Run business info migration")
parser.add_argument("--user_id", help="Target specific user ID")
args = parser.parse_args()
success = run_migration(args.user_id)
if success:
logger.success("🎉 Migration completed successfully!")

View File

@@ -7,29 +7,50 @@ This creates the scheduler_cumulative_stats table.
import sqlite3
import os
import sys
import argparse
# Get the database path
script_dir = os.path.dirname(os.path.abspath(__file__))
backend_dir = os.path.dirname(script_dir)
db_path = os.path.join(backend_dir, 'alwrity.db')
migration_path = os.path.join(backend_dir, 'database', 'migrations', 'create_scheduler_cumulative_stats.sql')
sys.path.insert(0, str(backend_dir))
if not os.path.exists(db_path):
print(f"❌ Database not found at {db_path}")
sys.exit(1)
from services.database import get_user_db_path
if not os.path.exists(migration_path):
print(f"❌ Migration file not found at {migration_path}")
sys.exit(1)
def run_migration(user_id=None):
if user_id:
db_path = get_user_db_path(user_id)
print(f"Targeting user database: {db_path}")
else:
print("❌ Error: user_id is required for migration.")
return False
try:
conn = sqlite3.connect(db_path)
with open(migration_path, 'r') as f:
conn.executescript(f.read())
conn.commit()
print("✅ Migration executed successfully")
conn.close()
except Exception as e:
print(f"❌ Error running migration: {e}")
sys.exit(1)
migration_path = os.path.join(backend_dir, 'database', 'migrations', 'create_scheduler_cumulative_stats.sql')
if not os.path.exists(db_path):
print(f"❌ Database not found at {db_path}")
return False
if not os.path.exists(migration_path):
print(f"❌ Migration file not found at {migration_path}")
return False
try:
conn = sqlite3.connect(db_path)
with open(migration_path, 'r') as f:
conn.executescript(f.read())
conn.commit()
print("✅ Migration executed successfully")
conn.close()
return True
except Exception as e:
print(f"❌ Error running migration: {e}")
return False
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Run cumulative stats migration")
parser.add_argument("--user_id", help="Target specific user ID")
args = parser.parse_args()
success = run_migration(args.user_id)
sys.exit(0 if success else 1)

View File

@@ -6,18 +6,21 @@ Adds consecutive_failures and failure_pattern columns to task tables.
import sqlite3
import os
import sys
import argparse
from pathlib import Path
# Add parent directory to path to import migration
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from services.database import get_user_db_path
def run_migration():
def run_migration(user_id=None):
"""Run the failure tracking migration."""
# Get database path
db_path = os.getenv('DATABASE_URL', 'sqlite:///alwrity.db')
# Extract path from SQLite URL if needed
if db_path.startswith('sqlite:///'):
db_path = db_path.replace('sqlite:///', '')
if user_id:
db_path = get_user_db_path(user_id)
print(f"Targeting user database: {db_path}")
else:
print("❌ Error: user_id is required for migration.")
return False
if not os.path.exists(db_path):
print(f"Database not found at {db_path}")
@@ -80,6 +83,10 @@ def run_migration():
return False
if __name__ == "__main__":
success = run_migration()
parser = argparse.ArgumentParser(description="Run failure tracking migration")
parser.add_argument("--user_id", help="Target specific user ID")
args = parser.parse_args()
success = run_migration(args.user_id)
sys.exit(0 if success else 1)

View File

@@ -7,6 +7,7 @@ This script should be run once to add the column to existing databases.
import os
import sys
import sqlite3
import argparse
from pathlib import Path
from loguru import logger
@@ -14,11 +15,18 @@ from loguru import logger
backend_dir = Path(__file__).parent.parent
sys.path.insert(0, str(backend_dir))
def run_migration():
from services.database import get_user_db_path
def run_migration(user_id=None):
"""Run the final_video_url column migration."""
try:
# Get the database path
db_path = backend_dir / "alwrity.db"
if user_id:
db_path = Path(get_user_db_path(user_id))
logger.info(f"Targeting user database: {db_path}")
else:
logger.error("❌ Error: user_id is required for migration.")
return False
logger.info(f"🔄 Starting final_video_url column migration...")
logger.info(f"📁 Database path: {db_path}")
@@ -86,6 +94,10 @@ def run_migration():
return False
if __name__ == "__main__":
success = run_migration()
parser = argparse.ArgumentParser(description="Run final_video_url migration")
parser.add_argument("--user_id", help="Target specific user ID")
args = parser.parse_args()
success = run_migration(args.user_id)
sys.exit(0 if success else 1)

View File

@@ -12,8 +12,16 @@ import os
import sys
import sqlite3
import json
import argparse
from pathlib import Path
# Add backend directory to path to import services
current_dir = os.path.dirname(os.path.abspath(__file__))
backend_dir = os.path.dirname(current_dir)
sys.path.append(backend_dir)
from services.database import get_user_db_path
def check_credentials_file():
"""Check if GSC credentials file exists and is valid."""
credentials_path = Path("gsc_credentials.json")
@@ -51,12 +59,18 @@ def check_credentials_file():
print(f"❌ Error reading credentials file: {e}")
return False
def check_database_tables():
def check_database_tables(user_id=None):
"""Check if GSC database tables exist."""
db_path = "alwrity.db"
if user_id:
db_path = get_user_db_path(user_id)
print(f"Targeting user database: {db_path}")
else:
print("❌ Error: user_id is required to check GSC tables.")
return False
if not os.path.exists(db_path):
print("❌ Database file not found!")
print(f"❌ Database file not found at {db_path}!")
print("📝 Please ensure the database is initialized.")
return False
@@ -104,11 +118,17 @@ def check_environment_variables():
print("✅ All required environment variables are set!")
return True
def create_database_tables():
def create_database_tables(user_id=None):
"""Create GSC database tables if they don't exist."""
db_path = "alwrity.db"
if user_id:
db_path = get_user_db_path(user_id)
else:
db_path = get_user_db_path('alwrity')
try:
# Ensure directory exists
os.makedirs(os.path.dirname(db_path), exist_ok=True)
with sqlite3.connect(db_path) as conn:
cursor = conn.cursor()
@@ -155,6 +175,10 @@ def create_database_tables():
def main():
"""Main setup function."""
parser = argparse.ArgumentParser(description="GSC Setup Script")
parser.add_argument("--user_id", help="Target specific user ID")
args = parser.parse_args()
print("🔧 Google Search Console Setup Check")
print("=" * 50)
@@ -176,9 +200,9 @@ def main():
# Check/create database tables
print("\n3. Checking database tables...")
if not check_database_tables():
if not check_database_tables(args.user_id):
print("📝 Creating missing database tables...")
if not create_database_tables():
if not create_database_tables(args.user_id):
all_good = False
# Summary

View File

@@ -3,28 +3,47 @@
import sqlite3
import os
import sys
import argparse
script_dir = os.path.dirname(os.path.abspath(__file__))
backend_dir = os.path.dirname(script_dir)
db_path = os.path.join(backend_dir, 'alwrity.db')
sys.path.insert(0, backend_dir)
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
from services.database import get_user_db_path
# Check if table exists
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='scheduler_cumulative_stats'")
result = cursor.fetchone()
print(f"Table exists: {result is not None}")
if result:
cursor.execute("SELECT * FROM scheduler_cumulative_stats WHERE id=1")
row = cursor.fetchone()
if row:
print(f"Row data: {row}")
def verify_stats(user_id=None):
if user_id:
db_path = get_user_db_path(user_id)
print(f"Targeting user database: {db_path}")
else:
print("Table exists but no row with id=1")
else:
print("Table does not exist")
print("❌ Error: user_id is required.")
return
conn.close()
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
# Check if table exists
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='scheduler_cumulative_stats'")
result = cursor.fetchone()
print(f"Table exists: {result is not None}")
if result:
cursor.execute("SELECT * FROM scheduler_cumulative_stats WHERE id=1")
row = cursor.fetchone()
if row:
print(f"Row data: {row}")
else:
print("Table exists but no row with id=1")
else:
print("Table does not exist")
conn.close()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Verify cumulative stats")
parser.add_argument("--user_id", help="Target specific user ID")
args = parser.parse_args()
verify_stats(args.user_id)