ALwrity version 0.5.6

2025-08-22 14:08:54 +05:30
parent 3f2f4d7b8c
commit 5d8d1cfb73
113 changed files with 28164 additions and 2968 deletions
--- a/backend/scripts/create_cache_table.py
+++ b/backend/scripts/create_cache_table.py
@@ -0,0 +1,140 @@
+"""
+Database migration script to create comprehensive user data cache table.
+Run this script to add the cache table to your database.
+"""
+
+import sys
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import sessionmaker
+from loguru import logger
+import os
+
+def create_cache_table():
+    """Create the comprehensive user data cache table."""
+    try:
+        # Get database URL from environment or use default
+        database_url = os.getenv('DATABASE_URL', 'sqlite:///alwrity.db')
+        
+        # Create engine
+        engine = create_engine(database_url)
+        
+        # Create session
+        SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+        db = SessionLocal()
+        
+        # SQL to create the cache table
+        create_table_sql = """
+        CREATE TABLE IF NOT EXISTS comprehensive_user_data_cache (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            user_id INTEGER NOT NULL,
+            strategy_id INTEGER,
+            data_hash VARCHAR(64) NOT NULL,
+            comprehensive_data JSON NOT NULL,
+            created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+            expires_at DATETIME NOT NULL,
+            last_accessed DATETIME DEFAULT CURRENT_TIMESTAMP,
+            access_count INTEGER DEFAULT 0
+        );
+        """
+        
+        # Create indexes
+        create_indexes_sql = [
+            "CREATE INDEX IF NOT EXISTS idx_user_strategy ON comprehensive_user_data_cache(user_id, strategy_id);",
+            "CREATE INDEX IF NOT EXISTS idx_expires_at ON comprehensive_user_data_cache(expires_at);",
+            "CREATE INDEX IF NOT EXISTS idx_data_hash ON comprehensive_user_data_cache(data_hash);"
+        ]
+        
+        # Execute table creation
+        logger.info("Creating comprehensive_user_data_cache table...")
+        db.execute(text(create_table_sql))
+        
+        # Execute index creation
+        logger.info("Creating indexes...")
+        for index_sql in create_indexes_sql:
+            db.execute(text(index_sql))
+        
+        # Commit changes
+        db.commit()
+        
+        # Verify table creation
+        result = db.execute(text("SELECT name FROM sqlite_master WHERE type='table' AND name='comprehensive_user_data_cache';"))
+        table_exists = result.fetchone()
+        
+        if table_exists:
+            logger.info("✅ Comprehensive user data cache table created successfully!")
+            
+            # Show table structure
+            result = db.execute(text("PRAGMA table_info(comprehensive_user_data_cache);"))
+            columns = result.fetchall()
+            
+            logger.info("Table structure:")
+            for column in columns:
+                logger.info(f"  - {column[1]} ({column[2]})")
+                
+        else:
+            logger.error("❌ Failed to create comprehensive_user_data_cache table")
+            return False
+            
+        db.close()
+        return True
+        
+    except Exception as e:
+        logger.error(f"❌ Error creating cache table: {str(e)}")
+        if 'db' in locals():
+            db.close()
+        return False
+
+def drop_cache_table():
+    """Drop the comprehensive user data cache table (for testing)."""
+    try:
+        # Get database URL from environment or use default
+        database_url = os.getenv('DATABASE_URL', 'sqlite:///alwrity.db')
+        
+        # Create engine
+        engine = create_engine(database_url)
+        
+        # Create session
+        SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+        db = SessionLocal()
+        
+        # Drop table
+        logger.info("Dropping comprehensive_user_data_cache table...")
+        db.execute(text("DROP TABLE IF EXISTS comprehensive_user_data_cache;"))
+        db.commit()
+        
+        logger.info("✅ Comprehensive user data cache table dropped successfully!")
+        db.close()
+        return True
+        
+    except Exception as e:
+        logger.error(f"❌ Error dropping cache table: {str(e)}")
+        if 'db' in locals():
+            db.close()
+        return False
+
+if __name__ == "__main__":
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Manage comprehensive user data cache table")
+    parser.add_argument("--action", choices=["create", "drop"], default="create", 
+                       help="Action to perform (create or drop table)")
+    
+    args = parser.parse_args()
+    
+    if args.action == "create":
+        success = create_cache_table()
+        if success:
+            logger.info("🎉 Cache table setup completed successfully!")
+        else:
+            logger.error("💥 Cache table setup failed!")
+            sys.exit(1)
+    elif args.action == "drop":
+        success = drop_cache_table()
+        if success:
+            logger.info("🗑️ Cache table dropped successfully!")
+        else:
+            logger.error("💥 Failed to drop cache table!")
+            sys.exit(1)
--- a/backend/scripts/create_monitoring_tables.py
+++ b/backend/scripts/create_monitoring_tables.py
@@ -1,47 +1,195 @@
-#!/usr/bin/env python3
 """
-Script to create monitoring tables in the database.
-Run this script to ensure all monitoring-related tables are created.
+Database migration script to create API monitoring tables.
+Run this script to add the monitoring tables to your database.
 """

 import sys
 import os
-
-# Add the backend directory to the Python path
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

-from services.database import init_database, get_db_session
-from models.monitoring_models import (
-    StrategyMonitoringPlan,
-    MonitoringTask,
-    TaskExecutionLog,
-    StrategyPerformanceMetrics,
-    StrategyActivationStatus
-)
-from models.enhanced_strategy_models import EnhancedContentStrategy
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import sessionmaker
 from loguru import logger
+import os

 def create_monitoring_tables():
-    """Create all monitoring-related tables"""
+    """Create the API monitoring tables."""
    try:
-        logger.info("Creating monitoring tables...")
+        # Get database URL from environment or use default
+        database_url = os.getenv('DATABASE_URL', 'sqlite:///alwrity.db')
        
-        # Initialize database with all models
-        init_database()
+        # Create engine
+        engine = create_engine(database_url)
        
-        logger.info("✅ Monitoring tables created successfully!")
+        # Create session
+        SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+        db = SessionLocal()
        
-        # Test database connection
-        db_session = get_db_session()
-        if db_session:
-            logger.info("✅ Database connection test successful!")
-            db_session.close()
-        else:
-            logger.warning("⚠️ Database connection test failed!")
+        # SQL to create the monitoring tables
+        create_tables_sql = [
+            """
+            CREATE TABLE IF NOT EXISTS api_requests (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                timestamp DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
+                path VARCHAR(500) NOT NULL,
+                method VARCHAR(10) NOT NULL,
+                status_code INTEGER NOT NULL,
+                duration FLOAT NOT NULL,
+                user_id VARCHAR(50),
+                cache_hit BOOLEAN,
+                request_size INTEGER,
+                response_size INTEGER,
+                user_agent VARCHAR(500),
+                ip_address VARCHAR(45)
+            );
+            """,
+            """
+            CREATE TABLE IF NOT EXISTS api_endpoint_stats (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                endpoint VARCHAR(500) NOT NULL UNIQUE,
+                total_requests INTEGER DEFAULT 0,
+                total_errors INTEGER DEFAULT 0,
+                total_duration FLOAT DEFAULT 0.0,
+                avg_duration FLOAT DEFAULT 0.0,
+                min_duration FLOAT,
+                max_duration FLOAT,
+                last_called DATETIME,
+                cache_hits INTEGER DEFAULT 0,
+                cache_misses INTEGER DEFAULT 0,
+                cache_hit_rate FLOAT DEFAULT 0.0,
+                updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
+            );
+            """,
+            """
+            CREATE TABLE IF NOT EXISTS system_health (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                timestamp DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
+                status VARCHAR(20) NOT NULL,
+                total_requests INTEGER DEFAULT 0,
+                total_errors INTEGER DEFAULT 0,
+                error_rate FLOAT DEFAULT 0.0,
+                avg_response_time FLOAT DEFAULT 0.0,
+                cache_hit_rate FLOAT DEFAULT 0.0,
+                active_endpoints INTEGER DEFAULT 0,
+                metrics JSON
+            );
+            """,
+            """
+            CREATE TABLE IF NOT EXISTS cache_performance (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                timestamp DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
+                cache_type VARCHAR(50) NOT NULL,
+                hits INTEGER DEFAULT 0,
+                misses INTEGER DEFAULT 0,
+                hit_rate FLOAT DEFAULT 0.0,
+                avg_response_time FLOAT DEFAULT 0.0,
+                total_requests INTEGER DEFAULT 0
+            );
+            """
+        ]
+        
+        # Create indexes
+        create_indexes_sql = [
+            "CREATE INDEX IF NOT EXISTS idx_api_requests_timestamp ON api_requests(timestamp);",
+            "CREATE INDEX IF NOT EXISTS idx_api_requests_path_method ON api_requests(path, method);",
+            "CREATE INDEX IF NOT EXISTS idx_api_requests_status_code ON api_requests(status_code);",
+            "CREATE INDEX IF NOT EXISTS idx_api_requests_user_id ON api_requests(user_id);",
+            "CREATE INDEX IF NOT EXISTS idx_api_endpoint_stats_endpoint ON api_endpoint_stats(endpoint);",
+            "CREATE INDEX IF NOT EXISTS idx_api_endpoint_stats_total_requests ON api_endpoint_stats(total_requests);",
+            "CREATE INDEX IF NOT EXISTS idx_api_endpoint_stats_avg_duration ON api_endpoint_stats(avg_duration);",
+            "CREATE INDEX IF NOT EXISTS idx_system_health_timestamp ON system_health(timestamp);",
+            "CREATE INDEX IF NOT EXISTS idx_system_health_status ON system_health(status);",
+            "CREATE INDEX IF NOT EXISTS idx_cache_performance_timestamp ON cache_performance(timestamp);",
+            "CREATE INDEX IF NOT EXISTS idx_cache_performance_cache_type ON cache_performance(cache_type);"
+        ]
+        
+        # Execute table creation
+        logger.info("Creating API monitoring tables...")
+        for table_sql in create_tables_sql:
+            db.execute(text(table_sql))
+        
+        # Execute index creation
+        logger.info("Creating indexes...")
+        for index_sql in create_indexes_sql:
+            db.execute(text(index_sql))
+        
+        # Commit changes
+        db.commit()
+        
+        # Verify table creation
+        tables_to_check = ['api_requests', 'api_endpoint_stats', 'system_health', 'cache_performance']
+        for table_name in tables_to_check:
+            result = db.execute(text(f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}';"))
+            table_exists = result.fetchone()
            
+            if table_exists:
+                logger.info(f"✅ {table_name} table created successfully!")
+            else:
+                logger.error(f"❌ Failed to create {table_name} table")
+                return False
+        
+        logger.info("🎉 All API monitoring tables created successfully!")
+        db.close()
+        return True
+        
    except Exception as e:
-        logger.error(f"❌ Error creating monitoring tables: {e}")
-        sys.exit(1)
+        logger.error(f"❌ Error creating monitoring tables: {str(e)}")
+        if 'db' in locals():
+            db.close()
+        return False
+
+def drop_monitoring_tables():
+    """Drop the API monitoring tables (for testing)."""
+    try:
+        # Get database URL from environment or use default
+        database_url = os.getenv('DATABASE_URL', 'sqlite:///alwrity.db')
+        
+        # Create engine
+        engine = create_engine(database_url)
+        
+        # Create session
+        SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+        db = SessionLocal()
+        
+        # Drop tables
+        tables_to_drop = ['api_requests', 'api_endpoint_stats', 'system_health', 'cache_performance']
+        logger.info("Dropping API monitoring tables...")
+        
+        for table_name in tables_to_drop:
+            db.execute(text(f"DROP TABLE IF EXISTS {table_name};"))
+        
+        db.commit()
+        
+        logger.info("✅ API monitoring tables dropped successfully!")
+        db.close()
+        return True
+        
+    except Exception as e:
+        logger.error(f"❌ Error dropping monitoring tables: {str(e)}")
+        if 'db' in locals():
+            db.close()
+        return False

 if __name__ == "__main__":
-    create_monitoring_tables()
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Manage API monitoring tables")
+    parser.add_argument("--action", choices=["create", "drop"], default="create", 
+                       help="Action to perform (create or drop tables)")
+    
+    args = parser.parse_args()
+    
+    if args.action == "create":
+        success = create_monitoring_tables()
+        if success:
+            logger.info("🎉 API monitoring tables setup completed successfully!")
+        else:
+            logger.error("💥 API monitoring tables setup failed!")
+            sys.exit(1)
+    elif args.action == "drop":
+        success = drop_monitoring_tables()
+        if success:
+            logger.info("🗑️ API monitoring tables dropped successfully!")
+        else:
+            logger.error("💥 Failed to drop API monitoring tables!")
+            sys.exit(1)
--- a/backend/scripts/generate_test_monitoring_data.py
+++ b/backend/scripts/generate_test_monitoring_data.py
@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""
+Generate Test Monitoring Data
+Creates sample API monitoring data to demonstrate the dashboard charts and animations.
+"""
+
+import sys
+import os
+import random
+from datetime import datetime, timedelta
+from sqlalchemy.orm import Session
+from sqlalchemy import text
+
+# Add the backend directory to the path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from services.database import get_db
+from models.api_monitoring import APIRequest, APIEndpointStats
+from loguru import logger
+
+def generate_test_monitoring_data():
+    """Generate test monitoring data for demonstration."""
+    logger.info("🎯 Generating test monitoring data...")
+    
+    db = next(get_db())
+    
+    try:
+        # Sample endpoints
+        endpoints = [
+            ("GET", "/api/content-planning/strategies"),
+            ("POST", "/api/content-planning/calendar-generation/start"),
+            ("GET", "/api/content-planning/monitoring/lightweight-stats"),
+            ("GET", "/api/content-planning/health"),
+            ("POST", "/api/content-planning/ai-analytics/analyze"),
+            ("GET", "/api/content-planning/gap-analysis"),
+            ("PUT", "/api/content-planning/strategies/1"),
+            ("DELETE", "/api/content-planning/strategies/2"),
+        ]
+        
+        # Generate requests for the last 30 minutes
+        now = datetime.utcnow()
+        start_time = now - timedelta(minutes=30)
+        
+        logger.info(f"📊 Generating data from {start_time} to {now}")
+        
+        for i in range(100):  # Generate 100 requests
+            # Random time within the last 30 minutes
+            timestamp = start_time + timedelta(
+                seconds=random.randint(0, 30 * 60)
+            )
+            
+            # Random endpoint
+            method, path = random.choice(endpoints)
+            
+            # Random status code (mostly 200, some errors)
+            if random.random() < 0.9:  # 90% success rate
+                status_code = 200
+            else:
+                status_code = random.choice([400, 401, 403, 404, 500, 502, 503])
+            
+            # Random duration (0.1 to 2.0 seconds)
+            duration = random.uniform(0.1, 2.0)
+            
+            # Random cache hit
+            cache_hit = random.choice([True, False, None])
+            
+            # Create API request
+            api_request = APIRequest(
+                path=path,
+                method=method,
+                status_code=status_code,
+                duration=duration,
+                user_id=f"user_{random.randint(1, 10)}",
+                cache_hit=cache_hit,
+                request_size=random.randint(100, 5000),
+                response_size=random.randint(500, 10000),
+                user_agent="Mozilla/5.0 (Test Browser)",
+                ip_address=f"192.168.1.{random.randint(1, 255)}",
+                timestamp=timestamp
+            )
+            db.add(api_request)
+        
+        # Generate endpoint stats
+        for method, path in endpoints:
+            endpoint_key = f"{method} {path}"
+            
+            # Check if stats already exist
+            existing_stats = db.query(APIEndpointStats).filter(
+                APIEndpointStats.endpoint == endpoint_key
+            ).first()
+            
+            if existing_stats:
+                # Update existing stats
+                total_requests = random.randint(50, 200)
+                total_errors = random.randint(0, total_requests // 10)
+                total_duration = random.uniform(10.0, 100.0)
+                
+                existing_stats.total_requests = total_requests
+                existing_stats.total_errors = total_errors
+                existing_stats.total_duration = total_duration
+                existing_stats.avg_duration = total_duration / total_requests
+                existing_stats.min_duration = random.uniform(0.05, 0.5)
+                existing_stats.max_duration = random.uniform(1.0, 3.0)
+                existing_stats.cache_hits = random.randint(0, total_requests // 2)
+                existing_stats.cache_misses = random.randint(0, total_requests // 3)
+                existing_stats.last_called = now
+                
+                if existing_stats.cache_hits + existing_stats.cache_misses > 0:
+                    existing_stats.cache_hit_rate = (
+                        existing_stats.cache_hits / 
+                        (existing_stats.cache_hits + existing_stats.cache_misses)
+                    ) * 100
+            else:
+                # Create new stats
+                total_requests = random.randint(50, 200)
+                total_errors = random.randint(0, total_requests // 10)
+                total_duration = random.uniform(10.0, 100.0)
+                cache_hits = random.randint(0, total_requests // 2)
+                cache_misses = random.randint(0, total_requests // 3)
+                
+                endpoint_stats = APIEndpointStats(
+                    endpoint=endpoint_key,
+                    total_requests=total_requests,
+                    total_errors=total_errors,
+                    total_duration=total_duration,
+                    avg_duration=total_duration / total_requests,
+                    min_duration=random.uniform(0.05, 0.5),
+                    max_duration=random.uniform(1.0, 3.0),
+                    cache_hits=cache_hits,
+                    cache_misses=cache_misses,
+                    cache_hit_rate=(cache_hits / (cache_hits + cache_misses)) * 100 if (cache_hits + cache_misses) > 0 else 0,
+                    last_called=now
+                )
+                db.add(endpoint_stats)
+        
+        db.commit()
+        logger.info("✅ Test monitoring data generated successfully!")
+        
+        # Show summary
+        total_requests = db.query(APIRequest).count()
+        total_errors = db.query(APIRequest).filter(APIRequest.status_code >= 400).count()
+        total_endpoints = db.query(APIEndpointStats).count()
+        
+        logger.info(f"📈 Generated {total_requests} API requests")
+        logger.info(f"❌ Generated {total_errors} error requests")
+        logger.info(f"🔗 Generated stats for {total_endpoints} endpoints")
+        
+        return True
+        
+    except Exception as e:
+        logger.error(f"❌ Error generating test data: {str(e)}")
+        db.rollback()
+        return False
+    finally:
+        db.close()
+
+def clear_test_data():
+    """Clear all test monitoring data."""
+    logger.info("🗑️ Clearing test monitoring data...")
+    
+    db = next(get_db())
+    
+    try:
+        # Clear all data
+        db.execute(text("DELETE FROM api_requests"))
+        db.execute(text("DELETE FROM api_endpoint_stats"))
+        db.execute(text("DELETE FROM system_health"))
+        db.execute(text("DELETE FROM cache_performance"))
+        
+        db.commit()
+        logger.info("✅ Test monitoring data cleared successfully!")
+        return True
+        
+    except Exception as e:
+        logger.error(f"❌ Error clearing test data: {str(e)}")
+        db.rollback()
+        return False
+    finally:
+        db.close()
+
+if __name__ == "__main__":
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Generate test monitoring data")
+    parser.add_argument("--action", choices=["generate", "clear"], default="generate",
+                       help="Action to perform (generate or clear test data)")
+    
+    args = parser.parse_args()
+    
+    if args.action == "generate":
+        success = generate_test_monitoring_data()
+        if success:
+            logger.info("🎉 Test data generation completed successfully!")
+        else:
+            logger.error("💥 Test data generation failed!")
+            sys.exit(1)
+    elif args.action == "clear":
+        success = clear_test_data()
+        if success:
+            logger.info("🗑️ Test data cleared successfully!")
+        else:
+            logger.error("💥 Failed to clear test data!")
+            sys.exit(1)