kunthawat/ALwrity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added new features to the project

Browse Source
This commit is contained in:
ajaysi
2025-06-30 07:49:48 +05:30
parent bbe56a364d
commit b21cbb68da
48 changed files with 19774 additions and 1889 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
lib/database/__init__.py Normal file
View File

@@ -0,0 +1,164 @@
"""
Database Package for ALwrity
============================
This package provides database models and services for managing data
in the ALwrity application, including Twitter-specific functionality.
Main Components:
- models.py: Core application database models
- twitter_models.py: Twitter-specific database models
- twitter_service.py: High-level Twitter database service
- twitter_init.py: Database initialization and management utilities
Usage:
# Initialize Twitter database
from lib.database import initialize_twitter_database
initialize_twitter_database()
# Use Twitter database service
from lib.database import twitter_db
user = twitter_db.create_or_update_user(user_data)
# Use Twitter models directly
from lib.database.twitter_models import TwitterUser, Tweet
"""
# Import core models
from .models import (
SEOData, ContentType, Platform, ScheduleStatus,
ContentItem, Schedule, create_engine, init_db, get_session
)
# Import Twitter-specific components
try:
from .twitter_models import (
# Models
TwitterUser, Tweet, ScheduledTweet, TwitterAnalytics,
TweetAnalytics, EngagementData, AudienceInsight,
HashtagPerformance, ContentTemplate, TwitterSettings,
# Enums and Data Classes
TwitterAccountType, TweetType, TweetStatus, EngagementType,
AnalyticsTimeframe, ContentCategory, TwitterCredentials, TweetMetrics,
# Database functions
get_twitter_engine, init_twitter_db, get_twitter_session,
create_twitter_user, update_user_metrics, create_tweet_record,
update_tweet_metrics, calculate_virality_score, get_user_analytics_summary
)
from .twitter_service import TwitterDatabaseService, twitter_db
from .twitter_init import (
TwitterDatabaseInitializer, initialize_twitter_database,
check_twitter_database_health
)
TWITTER_AVAILABLE = True
except ImportError as e:
# Twitter components not available (missing dependencies)
TWITTER_AVAILABLE = False
print(f"Warning: Twitter database components not available: {e}")
# Package metadata
__version__ = "1.0.0"
__author__ = "ALwrity Team"
# Export main components
__all__ = [
# Core models
'SEOData', 'ContentType', 'Platform', 'ScheduleStatus',
'ContentItem', 'Schedule', 'create_engine', 'init_db', 'get_session',
# Twitter availability flag
'TWITTER_AVAILABLE',
]
# Add Twitter exports if available
if TWITTER_AVAILABLE:
__all__.extend([
# Twitter Models
'TwitterUser', 'Tweet', 'ScheduledTweet', 'TwitterAnalytics',
'TweetAnalytics', 'EngagementData', 'AudienceInsight',
'HashtagPerformance', 'ContentTemplate', 'TwitterSettings',
# Twitter Enums and Data Classes
'TwitterAccountType', 'TweetType', 'TweetStatus', 'EngagementType',
'AnalyticsTimeframe', 'ContentCategory', 'TwitterCredentials', 'TweetMetrics',
# Twitter Database Functions
'get_twitter_engine', 'init_twitter_db', 'get_twitter_session',
'create_twitter_user', 'update_user_metrics', 'create_tweet_record',
'update_tweet_metrics', 'calculate_virality_score', 'get_user_analytics_summary',
# Twitter Service
'TwitterDatabaseService', 'twitter_db',
# Twitter Initialization
'TwitterDatabaseInitializer', 'initialize_twitter_database',
'check_twitter_database_health'
])
def setup_database(db_url: str = "sqlite:///alwrity.db", twitter_db_url: str = "sqlite:///twitter_data.db"):
"""
Setup both core and Twitter databases.
Args:
db_url: URL for the core database
twitter_db_url: URL for the Twitter database
Returns:
dict: Setup results
"""
results = {
'core_db': False,
'twitter_db': False,
'errors': []
}
try:
# Initialize core database
engine = create_engine(db_url)
init_db(engine)
results['core_db'] = True
except Exception as e:
results['errors'].append(f"Core database setup failed: {e}")
if TWITTER_AVAILABLE:
try:
# Initialize Twitter database
success = initialize_twitter_database(twitter_db_url)
results['twitter_db'] = success
if not success:
results['errors'].append("Twitter database initialization failed")
except Exception as e:
results['errors'].append(f"Twitter database setup failed: {e}")
else:
results['errors'].append("Twitter database components not available")
return results
def get_database_info():
"""
Get information about available database components.
Returns:
dict: Database component information
"""
info = {
'core_models_available': True,
'twitter_models_available': TWITTER_AVAILABLE,
'version': __version__
}
if TWITTER_AVAILABLE:
try:
# Get Twitter database stats if service is available
stats = twitter_db.get_database_stats()
info['twitter_stats'] = stats
except Exception as e:
info['twitter_stats_error'] = str(e)
return info

524
lib/database/twitter_init.py Normal file
View File

@@ -0,0 +1,524 @@
"""
Twitter Database Initialization and Migration Script
===================================================
This module provides utilities for initializing the Twitter database,
handling schema migrations, and managing database setup.
Features:
- Database initialization and table creation
- Schema migration utilities
- Data seeding for development/testing
- Database health checks and maintenance
"""
import os
import logging
from typing import Dict, Any, List, Optional
from datetime import datetime
import json
from pathlib import Path
from sqlalchemy import create_engine, text, inspect
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import SQLAlchemyError
from .twitter_models import (
Base, TwitterUser, Tweet, ScheduledTweet, TwitterAnalytics,
TweetAnalytics, EngagementData, AudienceInsight, HashtagPerformance,
ContentTemplate, TwitterSettings, TwitterAccountType, TweetType,
TweetStatus, EngagementType, AnalyticsTimeframe, ContentCategory
)
# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class TwitterDatabaseInitializer:
"""
Handles Twitter database initialization and management.
"""
def __init__(self, db_url: str = "sqlite:///twitter_data.db"):
"""Initialize the database initializer."""
self.db_url = db_url
self.engine = create_engine(db_url, echo=False)
self.SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=self.engine)
# Create database directory if using SQLite
if db_url.startswith('sqlite:///'):
db_path = db_url.replace('sqlite:///', '')
os.makedirs(os.path.dirname(os.path.abspath(db_path)), exist_ok=True)
def initialize_database(self, force_recreate: bool = False) -> bool:
"""
Initialize the Twitter database with all required tables.
Args:
force_recreate: If True, drop existing tables and recreate
Returns:
bool: True if successful, False otherwise
"""
try:
if force_recreate:
logger.info("Dropping existing tables...")
Base.metadata.drop_all(bind=self.engine)
logger.info("Creating Twitter database tables...")
Base.metadata.create_all(bind=self.engine)
# Verify tables were created
inspector = inspect(self.engine)
tables = inspector.get_table_names()
expected_tables = [
'twitter_users', 'tweets', 'scheduled_tweets', 'twitter_analytics',
'tweet_analytics', 'engagement_data', 'audience_insights',
'hashtag_performance', 'content_templates', 'twitter_settings'
]
missing_tables = [table for table in expected_tables if table not in tables]
if missing_tables:
logger.error(f"Missing tables: {missing_tables}")
return False
logger.info(f"Successfully created {len(tables)} tables")
# Create indexes for better performance
self._create_indexes()
# Seed initial data if needed
self._seed_initial_data()
logger.info("Twitter database initialization completed successfully")
return True
except Exception as e:
logger.error(f"Error initializing database: {e}")
return False
def _create_indexes(self):
"""Create database indexes for better query performance."""
try:
with self.engine.connect() as conn:
# User indexes
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_twitter_users_user_id ON twitter_users(user_id)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_twitter_users_twitter_user_id ON twitter_users(twitter_user_id)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_twitter_users_username ON twitter_users(username)"))
# Tweet indexes
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tweets_user_id ON tweets(user_id)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tweets_status ON tweets(status)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tweets_posted_at ON tweets(posted_at)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tweets_tweet_id ON tweets(tweet_id)"))
# Scheduled tweet indexes
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_scheduled_tweets_user_id ON scheduled_tweets(user_id)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_scheduled_tweets_status ON scheduled_tweets(status)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_scheduled_tweets_scheduled_time ON scheduled_tweets(scheduled_time)"))
# Analytics indexes
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_twitter_analytics_user_id ON twitter_analytics(user_id)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_twitter_analytics_date ON twitter_analytics(date)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_twitter_analytics_timeframe ON twitter_analytics(timeframe)"))
# Tweet analytics indexes
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tweet_analytics_tweet_id ON tweet_analytics(tweet_id)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tweet_analytics_recorded_at ON tweet_analytics(recorded_at)"))
# Engagement data indexes
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_engagement_data_tweet_id ON engagement_data(tweet_id)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_engagement_data_occurred_at ON engagement_data(occurred_at)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_engagement_data_type ON engagement_data(engagement_type)"))
# Hashtag performance indexes
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_hashtag_performance_user_id ON hashtag_performance(user_id)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_hashtag_performance_hashtag ON hashtag_performance(hashtag)"))
# Content template indexes
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_content_templates_user_id ON content_templates(user_id)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_content_templates_category ON content_templates(category)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS idx_content_templates_is_active ON content_templates(is_active)"))
conn.commit()
logger.info("Database indexes created successfully")
except Exception as e:
logger.error(f"Error creating indexes: {e}")
def _seed_initial_data(self):
"""Seed the database with initial data for development/testing."""
try:
session = self.SessionLocal()
# Check if we already have data
if session.query(TwitterUser).count() > 0:
logger.info("Database already contains data, skipping seeding")
session.close()
return
# Create sample content templates
sample_templates = [
{
'name': 'Daily Motivation',
'description': 'Motivational quotes and thoughts',
'template_text': 'Start your day with this thought: {quote} #motivation #success',
'category': ContentCategory.PERSONAL,
'variables': ['quote'],
'default_hashtags': ['#motivation', '#success', '#mindset'],
'ai_prompt': 'Generate an inspiring motivational quote',
'ai_tone': 'inspirational',
'ai_target_audience': 'professionals and entrepreneurs'
},
{
'name': 'Tech News Share',
'description': 'Template for sharing tech news',
'template_text': 'Interesting development in {topic}: {summary} {link} #tech #innovation',
'category': ContentCategory.EDUCATIONAL,
'variables': ['topic', 'summary', 'link'],
'default_hashtags': ['#tech', '#innovation', '#technology'],
'ai_prompt': 'Summarize this tech news in an engaging way',
'ai_tone': 'informative',
'ai_target_audience': 'tech enthusiasts and professionals'
},
{
'name': 'Question Engagement',
'description': 'Template for asking engaging questions',
'template_text': 'Quick question for my followers: {question} What do you think? #community #discussion',
'category': ContentCategory.QUESTION,
'variables': ['question'],
'default_hashtags': ['#community', '#discussion', '#question'],
'ai_prompt': 'Generate an engaging question for social media',
'ai_tone': 'conversational',
'ai_target_audience': 'general audience'
},
{
'name': 'Product Update',
'description': 'Template for product announcements',
'template_text': 'Excited to share: {update} {details} #product #update #announcement',
'category': ContentCategory.PROMOTIONAL,
'variables': ['update', 'details'],
'default_hashtags': ['#product', '#update', '#announcement'],
'ai_prompt': 'Write an exciting product update announcement',
'ai_tone': 'enthusiastic',
'ai_target_audience': 'customers and prospects'
}
]
# Note: We can't create templates without a user, so we'll skip this for now
# In a real scenario, templates would be created when users are added
session.close()
logger.info("Initial data seeding completed")
except Exception as e:
logger.error(f"Error seeding initial data: {e}")
def check_database_health(self) -> Dict[str, Any]:
"""
Check the health and status of the Twitter database.
Returns:
Dict containing health check results
"""
health_status = {
'status': 'healthy',
'timestamp': datetime.utcnow().isoformat(),
'tables': {},
'indexes': {},
'issues': []
}
try:
inspector = inspect(self.engine)
# Check table existence and row counts
expected_tables = [
'twitter_users', 'tweets', 'scheduled_tweets', 'twitter_analytics',
'tweet_analytics', 'engagement_data', 'audience_insights',
'hashtag_performance', 'content_templates', 'twitter_settings'
]
session = self.SessionLocal()
for table_name in expected_tables:
if table_name in inspector.get_table_names():
# Get row count
try:
result = session.execute(text(f"SELECT COUNT(*) FROM {table_name}"))
count = result.scalar()
health_status['tables'][table_name] = {
'exists': True,
'row_count': count
}
except Exception as e:
health_status['tables'][table_name] = {
'exists': True,
'row_count': 'error',
'error': str(e)
}
health_status['issues'].append(f"Error counting rows in {table_name}: {e}")
else:
health_status['tables'][table_name] = {'exists': False}
health_status['issues'].append(f"Missing table: {table_name}")
# Check indexes
for table_name in inspector.get_table_names():
indexes = inspector.get_indexes(table_name)
health_status['indexes'][table_name] = len(indexes)
session.close()
# Set overall status
if health_status['issues']:
health_status['status'] = 'issues_found'
return health_status
except Exception as e:
health_status['status'] = 'error'
health_status['error'] = str(e)
logger.error(f"Error checking database health: {e}")
return health_status
def backup_database(self, backup_path: str) -> bool:
"""
Create a backup of the database.
Args:
backup_path: Path where to save the backup
Returns:
bool: True if successful, False otherwise
"""
try:
if not self.db_url.startswith('sqlite:///'):
logger.error("Backup currently only supported for SQLite databases")
return False
# Get the database file path
db_file = self.db_url.replace('sqlite:///', '')
if not os.path.exists(db_file):
logger.error(f"Database file not found: {db_file}")
return False
# Create backup directory if it doesn't exist
os.makedirs(os.path.dirname(backup_path), exist_ok=True)
# Copy the database file
import shutil
shutil.copy2(db_file, backup_path)
logger.info(f"Database backed up to: {backup_path}")
return True
except Exception as e:
logger.error(f"Error backing up database: {e}")
return False
def restore_database(self, backup_path: str) -> bool:
"""
Restore database from a backup.
Args:
backup_path: Path to the backup file
Returns:
bool: True if successful, False otherwise
"""
try:
if not self.db_url.startswith('sqlite:///'):
logger.error("Restore currently only supported for SQLite databases")
return False
if not os.path.exists(backup_path):
logger.error(f"Backup file not found: {backup_path}")
return False
# Get the database file path
db_file = self.db_url.replace('sqlite:///', '')
# Copy the backup file to the database location
import shutil
shutil.copy2(backup_path, db_file)
logger.info(f"Database restored from: {backup_path}")
return True
except Exception as e:
logger.error(f"Error restoring database: {e}")
return False
def migrate_schema(self, migration_scripts: List[str]) -> bool:
"""
Apply schema migration scripts.
Args:
migration_scripts: List of SQL migration scripts
Returns:
bool: True if successful, False otherwise
"""
try:
with self.engine.connect() as conn:
# Create migration tracking table if it doesn't exist
conn.execute(text("""
CREATE TABLE IF NOT EXISTS schema_migrations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
migration_name TEXT NOT NULL UNIQUE,
applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
"""))
for script in migration_scripts:
# Check if migration was already applied
result = conn.execute(text(
"SELECT COUNT(*) FROM schema_migrations WHERE migration_name = :name"
), {"name": script})
if result.scalar() == 0:
# Apply migration
logger.info(f"Applying migration: {script}")
# Read and execute migration script
script_path = Path(script)
if script_path.exists():
with open(script_path, 'r') as f:
migration_sql = f.read()
conn.execute(text(migration_sql))
# Record migration as applied
conn.execute(text(
"INSERT INTO schema_migrations (migration_name) VALUES (:name)"
), {"name": script})
else:
logger.error(f"Migration script not found: {script}")
return False
else:
logger.info(f"Migration already applied: {script}")
conn.commit()
logger.info("Schema migration completed successfully")
return True
except Exception as e:
logger.error(f"Error applying schema migration: {e}")
return False
def cleanup_old_data(self, days: int = 90) -> Dict[str, int]:
"""
Clean up old data to maintain database performance.
Args:
days: Number of days to keep data for
Returns:
Dict with cleanup statistics
"""
try:
cutoff_date = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
cutoff_date = cutoff_date.replace(day=cutoff_date.day - days)
session = self.SessionLocal()
# Count records to be deleted
old_tweet_analytics = session.query(TweetAnalytics).filter(
TweetAnalytics.recorded_at < cutoff_date
).count()
old_engagement_data = session.query(EngagementData).filter(
EngagementData.occurred_at < cutoff_date
).count()
# Delete old records
session.query(TweetAnalytics).filter(
TweetAnalytics.recorded_at < cutoff_date
).delete()
session.query(EngagementData).filter(
EngagementData.occurred_at < cutoff_date
).delete()
session.commit()
session.close()
cleanup_stats = {
'tweet_analytics_deleted': old_tweet_analytics,
'engagement_data_deleted': old_engagement_data,
'cutoff_date': cutoff_date.isoformat()
}
logger.info(f"Cleanup completed: {cleanup_stats}")
return cleanup_stats
except Exception as e:
logger.error(f"Error during cleanup: {e}")
return {'error': str(e)}
def initialize_twitter_database(db_url: str = "sqlite:///twitter_data.db", force_recreate: bool = False) -> bool:
"""
Convenience function to initialize the Twitter database.
Args:
db_url: Database URL
force_recreate: Whether to recreate existing tables
Returns:
bool: True if successful, False otherwise
"""
initializer = TwitterDatabaseInitializer(db_url)
return initializer.initialize_database(force_recreate)
def check_twitter_database_health(db_url: str = "sqlite:///twitter_data.db") -> Dict[str, Any]:
"""
Convenience function to check Twitter database health.
Args:
db_url: Database URL
Returns:
Dict with health check results
"""
initializer = TwitterDatabaseInitializer(db_url)
return initializer.check_database_health()
if __name__ == "__main__":
# Command line interface for database management
import argparse
parser = argparse.ArgumentParser(description="Twitter Database Management")
parser.add_argument("--db-url", default="sqlite:///twitter_data.db", help="Database URL")
parser.add_argument("--init", action="store_true", help="Initialize database")
parser.add_argument("--force", action="store_true", help="Force recreate tables")
parser.add_argument("--health", action="store_true", help="Check database health")
parser.add_argument("--backup", help="Create database backup")
parser.add_argument("--restore", help="Restore from backup")
parser.add_argument("--cleanup", type=int, help="Cleanup data older than N days")
args = parser.parse_args()
initializer = TwitterDatabaseInitializer(args.db_url)
if args.init:
success = initializer.initialize_database(args.force)
print(f"Database initialization: {'SUCCESS' if success else 'FAILED'}")
if args.health:
health = initializer.check_database_health()
print(json.dumps(health, indent=2))
if args.backup:
success = initializer.backup_database(args.backup)
print(f"Database backup: {'SUCCESS' if success else 'FAILED'}")
if args.restore:
success = initializer.restore_database(args.restore)
print(f"Database restore: {'SUCCESS' if success else 'FAILED'}")
if args.cleanup:
stats = initializer.cleanup_old_data(args.cleanup)
print(f"Cleanup completed: {stats}")

791
lib/database/twitter_models.py Normal file
View File

@@ -0,0 +1,791 @@
"""
Twitter Database Models for ALwrity
===================================
This module defines SQLAlchemy models for storing Twitter-related data including:
- User profiles and authentication
- Tweet content and metadata
- Analytics and engagement metrics
- Scheduling and automation data
- Performance tracking and insights
This allows the application to store Twitter data locally and reduce API calls
while providing rich analytics and historical data to users.
"""
from sqlalchemy import (
create_engine, Column, Integer, String, Text, DateTime, Boolean, Float,
Enum, ForeignKey, JSON, BigInteger, Index, UniqueConstraint
)
from sqlalchemy.orm import declarative_base, relationship, sessionmaker
from datetime import datetime, timedelta
import enum
from dataclasses import dataclass
from typing import List, Dict, Any, Optional
import json
Base = declarative_base()
# --- ENUMS ---
class TwitterAccountType(enum.Enum):
PERSONAL = "personal"
BUSINESS = "business"
CREATOR = "creator"
BRAND = "brand"
class TweetType(enum.Enum):
ORIGINAL = "original"
REPLY = "reply"
RETWEET = "retweet"
QUOTE_TWEET = "quote_tweet"
THREAD = "thread"
class TweetStatus(enum.Enum):
DRAFT = "draft"
SCHEDULED = "scheduled"
POSTED = "posted"
FAILED = "failed"
DELETED = "deleted"
class EngagementType(enum.Enum):
LIKE = "like"
RETWEET = "retweet"
REPLY = "reply"
QUOTE_TWEET = "quote_tweet"
BOOKMARK = "bookmark"
IMPRESSION = "impression"
PROFILE_CLICK = "profile_click"
URL_CLICK = "url_click"
HASHTAG_CLICK = "hashtag_click"
MENTION_CLICK = "mention_click"
class AnalyticsTimeframe(enum.Enum):
HOURLY = "hourly"
DAILY = "daily"
WEEKLY = "weekly"
MONTHLY = "monthly"
class ContentCategory(enum.Enum):
EDUCATIONAL = "educational"
PROMOTIONAL = "promotional"
PERSONAL = "personal"
NEWS = "news"
ENTERTAINMENT = "entertainment"
QUESTION = "question"
POLL = "poll"
THREAD = "thread"
# --- DATACLASSES ---
@dataclass
class TwitterCredentials:
"""Dataclass for Twitter API credentials"""
api_key: str = ""
api_secret: str = ""
access_token: str = ""
access_token_secret: str = ""
bearer_token: str = ""
def to_dict(self) -> Dict[str, str]:
return {
'api_key': self.api_key,
'api_secret': self.api_secret,
'access_token': self.access_token,
'access_token_secret': self.access_token_secret,
'bearer_token': self.bearer_token
}
@classmethod
def from_dict(cls, data: Dict[str, str]) -> 'TwitterCredentials':
return cls(
api_key=data.get('api_key', ''),
api_secret=data.get('api_secret', ''),
access_token=data.get('access_token', ''),
access_token_secret=data.get('access_token_secret', ''),
bearer_token=data.get('bearer_token', '')
)
@dataclass
class TweetMetrics:
"""Dataclass for tweet performance metrics"""
likes: int = 0
retweets: int = 0
replies: int = 0
quotes: int = 0
bookmarks: int = 0
impressions: int = 0
profile_clicks: int = 0
url_clicks: int = 0
hashtag_clicks: int = 0
engagement_rate: float = 0.0
reach: int = 0
def to_dict(self) -> Dict[str, Any]:
return {
'likes': self.likes,
'retweets': self.retweets,
'replies': self.replies,
'quotes': self.quotes,
'bookmarks': self.bookmarks,
'impressions': self.impressions,
'profile_clicks': self.profile_clicks,
'url_clicks': self.url_clicks,
'hashtag_clicks': self.hashtag_clicks,
'engagement_rate': self.engagement_rate,
'reach': self.reach
}
# --- MODELS ---
class TwitterUser(Base):
"""
Stores Twitter user profile information and authentication data.
This reduces API calls for user profile information.
"""
__tablename__ = "twitter_users"
id = Column(Integer, primary_key=True)
user_id = Column(String, nullable=False, unique=True) # ALwrity user ID
twitter_user_id = Column(BigInteger, nullable=False, unique=True) # Twitter user ID
username = Column(String, nullable=False, index=True) # @username
display_name = Column(String, nullable=False)
bio = Column(Text)
location = Column(String)
website = Column(String)
profile_image_url = Column(String)
banner_image_url = Column(String)
# Account metrics
followers_count = Column(Integer, default=0)
following_count = Column(Integer, default=0)
tweet_count = Column(Integer, default=0)
listed_count = Column(Integer, default=0)
# Account details
account_type = Column(Enum(TwitterAccountType), default=TwitterAccountType.PERSONAL)
verified = Column(Boolean, default=False)
protected = Column(Boolean, default=False)
created_at_twitter = Column(DateTime) # When Twitter account was created
# Authentication and API data
credentials_encrypted = Column(Text) # Encrypted JSON of TwitterCredentials
api_rate_limit_remaining = Column(Integer, default=0)
api_rate_limit_reset = Column(DateTime)
last_api_call = Column(DateTime)
# Metadata
is_active = Column(Boolean, default=True)
last_sync = Column(DateTime, default=datetime.utcnow)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
# Relationships
tweets = relationship("Tweet", back_populates="user", cascade="all, delete-orphan")
analytics = relationship("TwitterAnalytics", back_populates="user", cascade="all, delete-orphan")
scheduled_tweets = relationship("ScheduledTweet", back_populates="user", cascade="all, delete-orphan")
engagement_data = relationship("EngagementData", back_populates="user", cascade="all, delete-orphan")
audience_insights = relationship("AudienceInsight", back_populates="user", cascade="all, delete-orphan")
# Indexes
__table_args__ = (
Index('idx_twitter_user_username', 'username'),
Index('idx_twitter_user_sync', 'last_sync'),
Index('idx_twitter_user_active', 'is_active'),
)
class Tweet(Base):
"""
Stores tweet content, metadata, and performance data.
Includes both posted tweets and drafts.
"""
__tablename__ = "tweets"
id = Column(Integer, primary_key=True)
user_id = Column(Integer, ForeignKey("twitter_users.id"), nullable=False)
tweet_id = Column(BigInteger, unique=True, index=True) # Twitter tweet ID (null for drafts)
# Content
text = Column(Text, nullable=False)
hashtags = Column(JSON, default=list) # List of hashtags
mentions = Column(JSON, default=list) # List of mentioned users
urls = Column(JSON, default=list) # List of URLs in tweet
media_urls = Column(JSON, default=list) # List of media URLs
# Tweet metadata
tweet_type = Column(Enum(TweetType), default=TweetType.ORIGINAL)
status = Column(Enum(TweetStatus), default=TweetStatus.DRAFT)
category = Column(Enum(ContentCategory))
# Engagement metrics (updated periodically)
likes_count = Column(Integer, default=0)
retweets_count = Column(Integer, default=0)
replies_count = Column(Integer, default=0)
quotes_count = Column(Integer, default=0)
bookmarks_count = Column(Integer, default=0)
impressions_count = Column(Integer, default=0)
# Performance metrics
engagement_rate = Column(Float, default=0.0)
reach = Column(Integer, default=0)
click_through_rate = Column(Float, default=0.0)
# AI and generation data
ai_generated = Column(Boolean, default=False)
ai_model_used = Column(String) # Which AI model generated this
ai_prompt = Column(Text) # Original prompt used
ai_confidence_score = Column(Float) # AI confidence in content quality
generation_metadata = Column(JSON, default=dict) # Additional AI metadata
# Scheduling and posting
scheduled_for = Column(DateTime)
posted_at = Column(DateTime)
last_metrics_update = Column(DateTime)
# Thread information
thread_id = Column(String) # For grouping thread tweets
thread_position = Column(Integer) # Position in thread (1, 2, 3...)
parent_tweet_id = Column(BigInteger) # For replies
# Metadata
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
# Relationships
user = relationship("TwitterUser", back_populates="tweets")
analytics = relationship("TweetAnalytics", back_populates="tweet", cascade="all, delete-orphan")
# Indexes
__table_args__ = (
Index('idx_tweet_user_status', 'user_id', 'status'),
Index('idx_tweet_posted_at', 'posted_at'),
Index('idx_tweet_engagement', 'engagement_rate'),
Index('idx_tweet_thread', 'thread_id'),
)
class ScheduledTweet(Base):
"""
Stores scheduled tweets with automation settings.
"""
__tablename__ = "scheduled_tweets"
id = Column(Integer, primary_key=True)
user_id = Column(Integer, ForeignKey("twitter_users.id"), nullable=False)
tweet_id = Column(Integer, ForeignKey("tweets.id"), nullable=False)
# Scheduling details
scheduled_time = Column(DateTime, nullable=False)
timezone = Column(String, default="UTC")
recurrence_pattern = Column(String) # cron-like pattern for recurring tweets
# Automation settings
auto_optimize_time = Column(Boolean, default=False) # AI-optimize posting time
auto_add_hashtags = Column(Boolean, default=False)
auto_add_emojis = Column(Boolean, default=False)
# Status and execution
status = Column(Enum(TweetStatus), default=TweetStatus.SCHEDULED)
attempts = Column(Integer, default=0)
last_attempt = Column(DateTime)
error_message = Column(Text)
# Metadata
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
# Relationships
user = relationship("TwitterUser", back_populates="scheduled_tweets")
tweet = relationship("Tweet")
# Indexes
__table_args__ = (
Index('idx_scheduled_time', 'scheduled_time'),
Index('idx_scheduled_status', 'status'),
)
class TwitterAnalytics(Base):
"""
Stores aggregated Twitter analytics data for users.
Updated periodically to track account performance over time.
"""
__tablename__ = "twitter_analytics"
id = Column(Integer, primary_key=True)
user_id = Column(Integer, ForeignKey("twitter_users.id"), nullable=False)
# Time period
date = Column(DateTime, nullable=False)
timeframe = Column(Enum(AnalyticsTimeframe), nullable=False)
# Account metrics
followers_gained = Column(Integer, default=0)
followers_lost = Column(Integer, default=0)
net_follower_change = Column(Integer, default=0)
following_change = Column(Integer, default=0)
# Content metrics
tweets_posted = Column(Integer, default=0)
total_impressions = Column(Integer, default=0)
total_engagements = Column(Integer, default=0)
total_likes = Column(Integer, default=0)
total_retweets = Column(Integer, default=0)
total_replies = Column(Integer, default=0)
total_quotes = Column(Integer, default=0)
# Performance metrics
average_engagement_rate = Column(Float, default=0.0)
top_tweet_id = Column(BigInteger) # Best performing tweet
top_tweet_engagement = Column(Integer, default=0)
# Audience metrics
profile_visits = Column(Integer, default=0)
mention_count = Column(Integer, default=0)
hashtag_performance = Column(JSON, default=dict) # Top hashtags and their performance
# Metadata
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
# Relationships
user = relationship("TwitterUser", back_populates="analytics")
# Indexes
__table_args__ = (
Index('idx_analytics_user_date', 'user_id', 'date'),
Index('idx_analytics_timeframe', 'timeframe'),
UniqueConstraint('user_id', 'date', 'timeframe', name='uq_user_date_timeframe'),
)
class TweetAnalytics(Base):
"""
Stores detailed analytics for individual tweets.
Updated periodically to track tweet performance over time.
"""
__tablename__ = "tweet_analytics"
id = Column(Integer, primary_key=True)
tweet_id = Column(Integer, ForeignKey("tweets.id"), nullable=False)
# Time period
recorded_at = Column(DateTime, nullable=False, default=datetime.utcnow)
# Engagement metrics
likes = Column(Integer, default=0)
retweets = Column(Integer, default=0)
replies = Column(Integer, default=0)
quotes = Column(Integer, default=0)
bookmarks = Column(Integer, default=0)
# Reach metrics
impressions = Column(Integer, default=0)
reach = Column(Integer, default=0)
profile_clicks = Column(Integer, default=0)
# Click metrics
url_clicks = Column(Integer, default=0)
hashtag_clicks = Column(Integer, default=0)
mention_clicks = Column(Integer, default=0)
media_views = Column(Integer, default=0)
# Calculated metrics
engagement_rate = Column(Float, default=0.0)
click_through_rate = Column(Float, default=0.0)
virality_score = Column(Float, default=0.0) # Custom metric for viral potential
# Metadata
created_at = Column(DateTime, default=datetime.utcnow)
# Relationships
tweet = relationship("Tweet", back_populates="analytics")
# Indexes
__table_args__ = (
Index('idx_tweet_analytics_recorded', 'recorded_at'),
Index('idx_tweet_analytics_engagement', 'engagement_rate'),
)
class EngagementData(Base):
"""
Stores individual engagement events for detailed analysis.
"""
__tablename__ = "engagement_data"
id = Column(Integer, primary_key=True)
user_id = Column(Integer, ForeignKey("twitter_users.id"), nullable=False)
tweet_id = Column(Integer, ForeignKey("tweets.id"))
# Engagement details
engagement_type = Column(Enum(EngagementType), nullable=False)
engaging_user_id = Column(BigInteger) # Twitter ID of user who engaged
engaging_username = Column(String)
# Metadata
occurred_at = Column(DateTime, nullable=False)
created_at = Column(DateTime, default=datetime.utcnow)
# Relationships
user = relationship("TwitterUser", back_populates="engagement_data")
tweet = relationship("Tweet")
# Indexes
__table_args__ = (
Index('idx_engagement_user_type', 'user_id', 'engagement_type'),
Index('idx_engagement_occurred', 'occurred_at'),
)
class AudienceInsight(Base):
"""
Stores audience demographics and behavior insights.
"""
__tablename__ = "audience_insights"
id = Column(Integer, primary_key=True)
user_id = Column(Integer, ForeignKey("twitter_users.id"), nullable=False)
# Time period
date = Column(DateTime, nullable=False)
# Demographics (aggregated data)
top_locations = Column(JSON, default=list) # Top follower locations
age_demographics = Column(JSON, default=dict) # Age distribution
gender_demographics = Column(JSON, default=dict) # Gender distribution
language_demographics = Column(JSON, default=dict) # Language distribution
# Behavior insights
most_active_hours = Column(JSON, default=list) # When audience is most active
top_interests = Column(JSON, default=list) # Audience interests
engagement_patterns = Column(JSON, default=dict) # How audience engages
# Content preferences
preferred_content_types = Column(JSON, default=dict)
top_hashtags_used = Column(JSON, default=list)
response_rate_by_content = Column(JSON, default=dict)
# Metadata
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
# Relationships
user = relationship("TwitterUser", back_populates="audience_insights")
# Indexes
__table_args__ = (
Index('idx_audience_user_date', 'user_id', 'date'),
)
class HashtagPerformance(Base):
"""
Tracks performance of hashtags used by the user.
"""
__tablename__ = "hashtag_performance"
id = Column(Integer, primary_key=True)
user_id = Column(Integer, ForeignKey("twitter_users.id"), nullable=False)
# Hashtag details
hashtag = Column(String, nullable=False, index=True)
usage_count = Column(Integer, default=0)
# Performance metrics
total_impressions = Column(Integer, default=0)
total_engagements = Column(Integer, default=0)
average_engagement_rate = Column(Float, default=0.0)
# Best performing tweet with this hashtag
best_tweet_id = Column(Integer, ForeignKey("tweets.id"))
best_tweet_engagement = Column(Integer, default=0)
# Time tracking
first_used = Column(DateTime)
last_used = Column(DateTime)
# Metadata
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
# Relationships
user = relationship("TwitterUser")
best_tweet = relationship("Tweet")
# Indexes
__table_args__ = (
Index('idx_hashtag_user_performance', 'user_id', 'average_engagement_rate'),
UniqueConstraint('user_id', 'hashtag', name='uq_user_hashtag'),
)
class ContentTemplate(Base):
"""
Stores reusable tweet templates and AI prompts.
"""
__tablename__ = "content_templates"
id = Column(Integer, primary_key=True)
user_id = Column(Integer, ForeignKey("twitter_users.id"), nullable=False)
# Template details
name = Column(String, nullable=False)
description = Column(Text)
template_text = Column(Text, nullable=False)
category = Column(Enum(ContentCategory))
# Template variables and settings
variables = Column(JSON, default=list) # List of template variables
default_hashtags = Column(JSON, default=list)
suggested_times = Column(JSON, default=list) # Best times to post this type
# AI settings
ai_prompt = Column(Text) # AI prompt for generating content
ai_tone = Column(String) # Tone for AI generation
ai_target_audience = Column(String)
# Usage tracking
usage_count = Column(Integer, default=0)
last_used = Column(DateTime)
average_performance = Column(Float, default=0.0)
# Metadata
is_active = Column(Boolean, default=True)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
# Relationships
user = relationship("TwitterUser")
# Indexes
__table_args__ = (
Index('idx_template_user_category', 'user_id', 'category'),
Index('idx_template_performance', 'average_performance'),
)
class TwitterSettings(Base):
"""
Stores user-specific Twitter settings and preferences.
"""
__tablename__ = "twitter_settings"
id = Column(Integer, primary_key=True)
user_id = Column(Integer, ForeignKey("twitter_users.id"), nullable=False, unique=True)
# Posting preferences
default_hashtags = Column(JSON, default=list)
auto_add_hashtags = Column(Boolean, default=False)
auto_add_emojis = Column(Boolean, default=False)
max_hashtags_per_tweet = Column(Integer, default=2)
# Scheduling preferences
preferred_posting_times = Column(JSON, default=list)
timezone = Column(String, default="UTC")
auto_optimize_timing = Column(Boolean, default=False)
# AI preferences
ai_tone_preference = Column(String, default="casual")
ai_target_audience = Column(String, default="general")
ai_creativity_level = Column(Float, default=0.7) # 0-1 scale
# Analytics preferences
analytics_frequency = Column(String, default="daily") # hourly, daily, weekly
track_competitor_hashtags = Column(JSON, default=list)
notification_preferences = Column(JSON, default=dict)
# Content preferences
content_categories = Column(JSON, default=list) # Preferred content types
avoid_topics = Column(JSON, default=list) # Topics to avoid
brand_keywords = Column(JSON, default=list) # Brand-related keywords
# Automation settings
auto_retweet_keywords = Column(JSON, default=list)
auto_like_keywords = Column(JSON, default=list)
auto_follow_back = Column(Boolean, default=False)
# Metadata
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
# Relationships
user = relationship("TwitterUser")
# --- DATABASE FUNCTIONS ---
def get_twitter_engine(db_url: str = "sqlite:///twitter_data.db"):
"""Create and return database engine for Twitter data."""
return create_engine(db_url, echo=False)
def init_twitter_db(engine):
"""Initialize Twitter database tables."""
Base.metadata.create_all(engine)
def get_twitter_session(engine):
"""Create and return database session for Twitter data."""
Session = sessionmaker(bind=engine)
return Session()
def create_twitter_user(session, user_data: Dict[str, Any]) -> TwitterUser:
"""Create a new Twitter user record."""
twitter_user = TwitterUser(
user_id=user_data['user_id'],
twitter_user_id=user_data['twitter_user_id'],
username=user_data['username'],
display_name=user_data['display_name'],
bio=user_data.get('bio', ''),
location=user_data.get('location', ''),
website=user_data.get('website', ''),
profile_image_url=user_data.get('profile_image_url', ''),
banner_image_url=user_data.get('banner_image_url', ''),
followers_count=user_data.get('followers_count', 0),
following_count=user_data.get('following_count', 0),
tweet_count=user_data.get('tweet_count', 0),
verified=user_data.get('verified', False),
protected=user_data.get('protected', False),
created_at_twitter=user_data.get('created_at_twitter'),
credentials_encrypted=user_data.get('credentials_encrypted', ''),
)
session.add(twitter_user)
session.commit()
return twitter_user
def update_user_metrics(session, user_id: int, metrics: Dict[str, Any]):
"""Update user metrics from Twitter API."""
user = session.query(TwitterUser).filter_by(id=user_id).first()
if user:
user.followers_count = metrics.get('followers_count', user.followers_count)
user.following_count = metrics.get('following_count', user.following_count)
user.tweet_count = metrics.get('tweet_count', user.tweet_count)
user.last_sync = datetime.utcnow()
session.commit()
def create_tweet_record(session, tweet_data: Dict[str, Any]) -> Tweet:
"""Create a new tweet record."""
# Handle both 'text' and 'content' field names for compatibility
text_content = tweet_data.get('text') or tweet_data.get('content')
if not text_content:
raise ValueError("Tweet must have either 'text' or 'content' field")
tweet = Tweet(
user_id=tweet_data['user_id'],
tweet_id=tweet_data.get('tweet_id'),
text=text_content,
hashtags=tweet_data.get('hashtags', []),
mentions=tweet_data.get('mentions', []),
urls=tweet_data.get('urls', []),
media_urls=tweet_data.get('media_urls', []),
tweet_type=TweetType(tweet_data.get('tweet_type', 'original')),
status=TweetStatus(tweet_data.get('status', 'draft')),
category=ContentCategory(tweet_data['category']) if tweet_data.get('category') else None,
ai_generated=tweet_data.get('ai_generated', False),
ai_model_used=tweet_data.get('ai_model_used'),
ai_prompt=tweet_data.get('ai_prompt'),
ai_confidence_score=tweet_data.get('ai_confidence_score'),
generation_metadata=tweet_data.get('generation_metadata', {}),
scheduled_for=tweet_data.get('scheduled_for'),
posted_at=tweet_data.get('posted_at'),
thread_id=tweet_data.get('thread_id'),
thread_position=tweet_data.get('thread_position'),
parent_tweet_id=tweet_data.get('parent_tweet_id'),
)
session.add(tweet)
session.commit()
return tweet
def update_tweet_metrics(session, tweet_id: int, metrics: TweetMetrics):
"""Update tweet metrics from Twitter API."""
tweet = session.query(Tweet).filter_by(id=tweet_id).first()
if tweet:
tweet.likes_count = metrics.likes
tweet.retweets_count = metrics.retweets
tweet.replies_count = metrics.replies
tweet.quotes_count = metrics.quotes
tweet.bookmarks_count = metrics.bookmarks
tweet.impressions_count = metrics.impressions
tweet.engagement_rate = metrics.engagement_rate
tweet.reach = metrics.reach
tweet.last_metrics_update = datetime.utcnow()
session.commit()
# Also create analytics record
analytics = TweetAnalytics(
tweet_id=tweet_id,
likes=metrics.likes,
retweets=metrics.retweets,
replies=metrics.replies,
quotes=metrics.quotes,
bookmarks=metrics.bookmarks,
impressions=metrics.impressions,
reach=metrics.reach,
engagement_rate=metrics.engagement_rate,
click_through_rate=metrics.url_clicks / max(metrics.impressions, 1) * 100,
virality_score=calculate_virality_score(metrics)
)
session.add(analytics)
session.commit()
def calculate_virality_score(metrics: TweetMetrics) -> float:
"""Calculate a custom virality score based on engagement metrics."""
if metrics.impressions == 0:
return 0.0
# Weight different engagement types
engagement_score = (
metrics.likes * 1.0 +
metrics.retweets * 3.0 + # Retweets are more valuable
metrics.replies * 2.0 +
metrics.quotes * 2.5 +
metrics.bookmarks * 1.5
)
# Normalize by impressions and scale
virality = (engagement_score / metrics.impressions) * 100
return min(virality, 100.0) # Cap at 100
def get_user_analytics_summary(session, user_id: int, days: int = 30) -> Dict[str, Any]:
"""Get analytics summary for a user over specified days."""
from sqlalchemy import func
start_date = datetime.utcnow() - timedelta(days=days)
# Get tweet metrics
tweet_stats = session.query(
func.count(Tweet.id).label('total_tweets'),
func.avg(Tweet.engagement_rate).label('avg_engagement'),
func.sum(Tweet.likes_count).label('total_likes'),
func.sum(Tweet.retweets_count).label('total_retweets'),
func.sum(Tweet.impressions_count).label('total_impressions')
).filter(
Tweet.user_id == user_id,
Tweet.posted_at >= start_date,
Tweet.status == TweetStatus.POSTED
).first()
# Get follower growth
user = session.query(TwitterUser).filter_by(id=user_id).first()
return {
'total_tweets': tweet_stats.total_tweets or 0,
'average_engagement_rate': float(tweet_stats.avg_engagement or 0),
'total_likes': tweet_stats.total_likes or 0,
'total_retweets': tweet_stats.total_retweets or 0,
'total_impressions': tweet_stats.total_impressions or 0,
'current_followers': user.followers_count if user else 0,
'period_days': days
}
# Export all models and functions
__all__ = [
# Models
'TwitterUser', 'Tweet', 'ScheduledTweet', 'TwitterAnalytics', 'TweetAnalytics',
'EngagementData', 'AudienceInsight', 'HashtagPerformance', 'ContentTemplate',
'TwitterSettings',
# Enums
'TwitterAccountType', 'TweetType', 'TweetStatus', 'EngagementType',
'AnalyticsTimeframe', 'ContentCategory',
# Dataclasses
'TwitterCredentials', 'TweetMetrics',
# Functions
'get_twitter_engine', 'init_twitter_db', 'get_twitter_session',
'create_twitter_user', 'update_user_metrics', 'create_tweet_record',
'update_tweet_metrics', 'calculate_virality_score', 'get_user_analytics_summary'
]

766
lib/database/twitter_service.py Normal file
View File

@@ -0,0 +1,766 @@
"""
Twitter Database Service Layer
=============================
This module provides high-level service functions for managing Twitter data
in the database. It acts as an interface between the application and the
database models, providing convenient methods for common operations.
Key Features:
- User profile management and synchronization
- Tweet creation, updating, and analytics tracking
- Scheduled tweet management
- Analytics data aggregation and reporting
- Hashtag performance tracking
- Audience insights management
"""
import logging
from typing import Dict, List, Any, Optional, Tuple
from datetime import datetime, timedelta
from sqlalchemy.orm import Session
from sqlalchemy import func, desc, and_, or_
import json
from cryptography.fernet import Fernet
import os
from .twitter_models import (
TwitterUser, Tweet, ScheduledTweet, TwitterAnalytics, TweetAnalytics,
EngagementData, AudienceInsight, HashtagPerformance, ContentTemplate,
TwitterSettings, TwitterCredentials, TweetMetrics,
TwitterAccountType, TweetType, TweetStatus, EngagementType,
AnalyticsTimeframe, ContentCategory,
get_twitter_engine, init_twitter_db, get_twitter_session,
create_twitter_user, update_user_metrics, create_tweet_record,
update_tweet_metrics, calculate_virality_score, get_user_analytics_summary
)
# Configure logging
logger = logging.getLogger(__name__)
class TwitterDatabaseService:
"""
High-level service for managing Twitter data in the database.
"""
def __init__(self, db_url: str = "sqlite:///twitter_data.db", encryption_key: Optional[str] = None):
"""Initialize the Twitter database service."""
self.engine = get_twitter_engine(db_url)
self.encryption_key = encryption_key or self._get_or_create_encryption_key()
self.cipher = Fernet(self.encryption_key.encode() if isinstance(self.encryption_key, str) else self.encryption_key)
# Initialize database
init_twitter_db(self.engine)
logger.info("Twitter database service initialized")
def _get_or_create_encryption_key(self) -> str:
"""Get or create encryption key for sensitive data."""
key_file = "twitter_encryption.key"
if os.path.exists(key_file):
with open(key_file, 'rb') as f:
return f.read()
else:
key = Fernet.generate_key()
with open(key_file, 'wb') as f:
f.write(key)
return key
def _encrypt_credentials(self, credentials: TwitterCredentials) -> str:
"""Encrypt Twitter credentials for secure storage."""
credentials_json = json.dumps(credentials.to_dict())
encrypted = self.cipher.encrypt(credentials_json.encode())
return encrypted.decode()
def _decrypt_credentials(self, encrypted_credentials: str) -> TwitterCredentials:
"""Decrypt Twitter credentials from storage."""
try:
decrypted = self.cipher.decrypt(encrypted_credentials.encode())
credentials_dict = json.loads(decrypted.decode())
return TwitterCredentials.from_dict(credentials_dict)
except Exception as e:
logger.error(f"Failed to decrypt credentials: {e}")
return TwitterCredentials()
def get_session(self) -> Session:
"""Get a database session."""
return get_twitter_session(self.engine)
# --- USER MANAGEMENT ---
def create_or_update_user(self, user_data: Dict[str, Any]) -> TwitterUser:
"""Create a new Twitter user or update existing one."""
session = self.get_session()
try:
# Check if user already exists
existing_user = session.query(TwitterUser).filter_by(
user_id=user_data['user_id']
).first()
if existing_user:
# Update existing user
for key, value in user_data.items():
if hasattr(existing_user, key) and key != 'id':
setattr(existing_user, key, value)
existing_user.updated_at = datetime.utcnow()
session.commit()
logger.info(f"Updated Twitter user: {existing_user.username}")
return existing_user
else:
# Create new user
twitter_user = create_twitter_user(session, user_data)
logger.info(f"Created new Twitter user: {twitter_user.username}")
return twitter_user
except Exception as e:
session.rollback()
logger.error(f"Error creating/updating user: {e}")
raise
finally:
session.close()
def save_user_credentials(self, user_id: str, credentials: TwitterCredentials) -> bool:
"""Save encrypted Twitter credentials for a user."""
session = self.get_session()
try:
user = session.query(TwitterUser).filter_by(user_id=user_id).first()
if user:
encrypted_creds = self._encrypt_credentials(credentials)
user.credentials_encrypted = encrypted_creds
user.updated_at = datetime.utcnow()
session.commit()
logger.info(f"Saved credentials for user: {user.username}")
return True
else:
logger.error(f"User not found: {user_id}")
return False
except Exception as e:
session.rollback()
logger.error(f"Error saving credentials: {e}")
return False
finally:
session.close()
def get_user_credentials(self, user_id: str) -> Optional[TwitterCredentials]:
"""Get decrypted Twitter credentials for a user."""
session = self.get_session()
try:
user = session.query(TwitterUser).filter_by(user_id=user_id).first()
if user and user.credentials_encrypted:
return self._decrypt_credentials(user.credentials_encrypted)
return None
except Exception as e:
logger.error(f"Error getting credentials: {e}")
return None
finally:
session.close()
def get_user_by_id(self, user_id: str) -> Optional[TwitterUser]:
"""Get Twitter user by ALwrity user ID."""
session = self.get_session()
try:
return session.query(TwitterUser).filter_by(user_id=user_id).first()
finally:
session.close()
def get_user_by_twitter_id(self, twitter_user_id: int) -> Optional[TwitterUser]:
"""Get Twitter user by Twitter user ID."""
session = self.get_session()
try:
return session.query(TwitterUser).filter_by(twitter_user_id=twitter_user_id).first()
finally:
session.close()
def update_user_profile(self, user_id: str, profile_data: Dict[str, Any]) -> bool:
"""Update user profile information from Twitter API."""
session = self.get_session()
try:
user = session.query(TwitterUser).filter_by(user_id=user_id).first()
if user:
update_user_metrics(session, user.id, profile_data)
logger.info(f"Updated profile for user: {user.username}")
return True
return False
except Exception as e:
session.rollback()
logger.error(f"Error updating user profile: {e}")
return False
finally:
session.close()
# --- TWEET MANAGEMENT ---
def save_tweet(self, tweet_data: Dict[str, Any]) -> Tweet:
"""Save a tweet to the database."""
session = self.get_session()
try:
tweet = create_tweet_record(session, tweet_data)
logger.info(f"Saved tweet: {tweet.id}")
return tweet
except Exception as e:
session.rollback()
logger.error(f"Error saving tweet: {e}")
raise
finally:
session.close()
def update_tweet_status(self, tweet_id: int, status: TweetStatus, twitter_tweet_id: Optional[int] = None) -> bool:
"""Update tweet status (e.g., from draft to posted)."""
session = self.get_session()
try:
tweet = session.query(Tweet).filter_by(id=tweet_id).first()
if tweet:
tweet.status = status
if twitter_tweet_id:
tweet.tweet_id = twitter_tweet_id
if status == TweetStatus.POSTED:
tweet.posted_at = datetime.utcnow()
tweet.updated_at = datetime.utcnow()
session.commit()
logger.info(f"Updated tweet {tweet_id} status to {status.value}")
return True
return False
except Exception as e:
session.rollback()
logger.error(f"Error updating tweet status: {e}")
return False
finally:
session.close()
def get_user_tweets(self, user_id: str, status: Optional[TweetStatus] = None, limit: int = 50) -> List[Tweet]:
"""Get tweets for a user, optionally filtered by status."""
session = self.get_session()
try:
user = session.query(TwitterUser).filter_by(user_id=user_id).first()
if not user:
return []
query = session.query(Tweet).filter_by(user_id=user.id)
if status:
query = query.filter_by(status=status)
return query.order_by(desc(Tweet.created_at)).limit(limit).all()
finally:
session.close()
def get_tweet_by_id(self, tweet_id: int) -> Optional[Tweet]:
"""Get tweet by database ID."""
session = self.get_session()
try:
return session.query(Tweet).filter_by(id=tweet_id).first()
finally:
session.close()
def get_tweet_by_twitter_id(self, twitter_tweet_id: int) -> Optional[Tweet]:
"""Get tweet by Twitter tweet ID."""
session = self.get_session()
try:
return session.query(Tweet).filter_by(tweet_id=twitter_tweet_id).first()
finally:
session.close()
def update_tweet_analytics(self, tweet_id: int, metrics: TweetMetrics) -> bool:
"""Update tweet analytics from Twitter API."""
session = self.get_session()
try:
update_tweet_metrics(session, tweet_id, metrics)
logger.info(f"Updated analytics for tweet: {tweet_id}")
return True
except Exception as e:
session.rollback()
logger.error(f"Error updating tweet analytics: {e}")
return False
finally:
session.close()
def get_top_performing_tweets(self, user_id: str, days: int = 30, limit: int = 10) -> List[Tweet]:
"""Get top performing tweets for a user."""
session = self.get_session()
try:
user = session.query(TwitterUser).filter_by(user_id=user_id).first()
if not user:
return []
start_date = datetime.utcnow() - timedelta(days=days)
return session.query(Tweet).filter(
and_(
Tweet.user_id == user.id,
Tweet.status == TweetStatus.POSTED,
Tweet.posted_at >= start_date
)
).order_by(desc(Tweet.engagement_rate)).limit(limit).all()
finally:
session.close()
# --- SCHEDULED TWEETS ---
def schedule_tweet(self, tweet_id: int, scheduled_time: datetime, settings: Dict[str, Any] = None) -> ScheduledTweet:
"""Schedule a tweet for posting."""
session = self.get_session()
try:
tweet = session.query(Tweet).filter_by(id=tweet_id).first()
if not tweet:
raise ValueError(f"Tweet {tweet_id} not found")
scheduled_tweet = ScheduledTweet(
user_id=tweet.user_id,
tweet_id=tweet_id,
scheduled_time=scheduled_time,
timezone=settings.get('timezone', 'UTC'),
auto_optimize_time=settings.get('auto_optimize_time', False),
auto_add_hashtags=settings.get('auto_add_hashtags', False),
auto_add_emojis=settings.get('auto_add_emojis', False)
)
session.add(scheduled_tweet)
# Update tweet status
tweet.status = TweetStatus.SCHEDULED
tweet.scheduled_for = scheduled_time
session.commit()
logger.info(f"Scheduled tweet {tweet_id} for {scheduled_time}")
return scheduled_tweet
except Exception as e:
session.rollback()
logger.error(f"Error scheduling tweet: {e}")
raise
finally:
session.close()
def get_pending_scheduled_tweets(self, user_id: Optional[str] = None) -> List[ScheduledTweet]:
"""Get tweets scheduled for posting."""
session = self.get_session()
try:
query = session.query(ScheduledTweet).filter(
and_(
ScheduledTweet.status == TweetStatus.SCHEDULED,
ScheduledTweet.scheduled_time <= datetime.utcnow()
)
)
if user_id:
user = session.query(TwitterUser).filter_by(user_id=user_id).first()
if user:
query = query.filter_by(user_id=user.id)
return query.order_by(ScheduledTweet.scheduled_time).all()
finally:
session.close()
def mark_scheduled_tweet_posted(self, scheduled_tweet_id: int, twitter_tweet_id: int) -> bool:
"""Mark a scheduled tweet as posted."""
session = self.get_session()
try:
scheduled_tweet = session.query(ScheduledTweet).filter_by(id=scheduled_tweet_id).first()
if scheduled_tweet:
scheduled_tweet.status = TweetStatus.POSTED
# Update the associated tweet
tweet = session.query(Tweet).filter_by(id=scheduled_tweet.tweet_id).first()
if tweet:
tweet.status = TweetStatus.POSTED
tweet.tweet_id = twitter_tweet_id
tweet.posted_at = datetime.utcnow()
session.commit()
logger.info(f"Marked scheduled tweet {scheduled_tweet_id} as posted")
return True
return False
except Exception as e:
session.rollback()
logger.error(f"Error marking scheduled tweet as posted: {e}")
return False
finally:
session.close()
# --- ANALYTICS ---
def save_daily_analytics(self, user_id: str, analytics_data: Dict[str, Any]) -> TwitterAnalytics:
"""Save daily analytics data for a user."""
session = self.get_session()
try:
user = session.query(TwitterUser).filter_by(user_id=user_id).first()
if not user:
raise ValueError(f"User {user_id} not found")
# Check if analytics for today already exist
today = datetime.utcnow().date()
existing = session.query(TwitterAnalytics).filter(
and_(
TwitterAnalytics.user_id == user.id,
func.date(TwitterAnalytics.date) == today,
TwitterAnalytics.timeframe == AnalyticsTimeframe.DAILY
)
).first()
if existing:
# Update existing record
for key, value in analytics_data.items():
if hasattr(existing, key):
setattr(existing, key, value)
existing.updated_at = datetime.utcnow()
session.commit()
return existing
else:
# Create new record
analytics = TwitterAnalytics(
user_id=user.id,
date=datetime.utcnow(),
timeframe=AnalyticsTimeframe.DAILY,
**analytics_data
)
session.add(analytics)
session.commit()
logger.info(f"Saved daily analytics for user: {user.username}")
return analytics
except Exception as e:
session.rollback()
logger.error(f"Error saving analytics: {e}")
raise
finally:
session.close()
def get_analytics_summary(self, user_id: str, days: int = 30) -> Dict[str, Any]:
"""Get comprehensive analytics summary for a user."""
session = self.get_session()
try:
return get_user_analytics_summary(session, user_id, days)
finally:
session.close()
def get_engagement_trends(self, user_id: str, days: int = 30) -> List[Dict[str, Any]]:
"""Get engagement trends over time."""
session = self.get_session()
try:
user = session.query(TwitterUser).filter_by(user_id=user_id).first()
if not user:
return []
start_date = datetime.utcnow() - timedelta(days=days)
analytics = session.query(TwitterAnalytics).filter(
and_(
TwitterAnalytics.user_id == user.id,
TwitterAnalytics.date >= start_date,
TwitterAnalytics.timeframe == AnalyticsTimeframe.DAILY
)
).order_by(TwitterAnalytics.date).all()
return [
{
'date': a.date.isoformat(),
'engagement_rate': a.average_engagement_rate,
'total_engagements': a.total_engagements,
'impressions': a.total_impressions,
'followers_change': a.net_follower_change
}
for a in analytics
]
finally:
session.close()
# --- HASHTAG PERFORMANCE ---
def track_hashtag_performance(self, user_id: str, hashtag: str, tweet_id: int, engagement_metrics: Dict[str, Any]) -> bool:
"""Track performance of a hashtag."""
session = self.get_session()
try:
user = session.query(TwitterUser).filter_by(user_id=user_id).first()
if not user:
return False
# Get or create hashtag performance record
hashtag_perf = session.query(HashtagPerformance).filter(
and_(
HashtagPerformance.user_id == user.id,
HashtagPerformance.hashtag == hashtag
)
).first()
if hashtag_perf:
# Update existing record
hashtag_perf.usage_count += 1
hashtag_perf.total_impressions += engagement_metrics.get('impressions', 0)
hashtag_perf.total_engagements += engagement_metrics.get('engagements', 0)
hashtag_perf.last_used = datetime.utcnow()
# Update average engagement rate
if hashtag_perf.usage_count > 0:
hashtag_perf.average_engagement_rate = (
hashtag_perf.total_engagements / hashtag_perf.total_impressions * 100
if hashtag_perf.total_impressions > 0 else 0
)
# Update best performing tweet if this one is better
current_engagement = engagement_metrics.get('engagements', 0)
if current_engagement > hashtag_perf.best_tweet_engagement:
hashtag_perf.best_tweet_id = tweet_id
hashtag_perf.best_tweet_engagement = current_engagement
else:
# Create new record
hashtag_perf = HashtagPerformance(
user_id=user.id,
hashtag=hashtag,
usage_count=1,
total_impressions=engagement_metrics.get('impressions', 0),
total_engagements=engagement_metrics.get('engagements', 0),
average_engagement_rate=(
engagement_metrics.get('engagements', 0) /
max(engagement_metrics.get('impressions', 1), 1) * 100
),
best_tweet_id=tweet_id,
best_tweet_engagement=engagement_metrics.get('engagements', 0),
first_used=datetime.utcnow(),
last_used=datetime.utcnow()
)
session.add(hashtag_perf)
session.commit()
return True
except Exception as e:
session.rollback()
logger.error(f"Error tracking hashtag performance: {e}")
return False
finally:
session.close()
def get_top_hashtags(self, user_id: str, limit: int = 10) -> List[HashtagPerformance]:
"""Get top performing hashtags for a user."""
session = self.get_session()
try:
user = session.query(TwitterUser).filter_by(user_id=user_id).first()
if not user:
return []
return session.query(HashtagPerformance).filter_by(
user_id=user.id
).order_by(desc(HashtagPerformance.average_engagement_rate)).limit(limit).all()
finally:
session.close()
# --- CONTENT TEMPLATES ---
def save_content_template(self, user_id: str, template_data: Dict[str, Any]) -> ContentTemplate:
"""Save a content template."""
session = self.get_session()
try:
user = session.query(TwitterUser).filter_by(user_id=user_id).first()
if not user:
raise ValueError(f"User {user_id} not found")
template = ContentTemplate(
user_id=user.id,
name=template_data['name'],
description=template_data.get('description', ''),
template_text=template_data['template_text'],
category=ContentCategory(template_data['category']) if template_data.get('category') else None,
variables=template_data.get('variables', []),
default_hashtags=template_data.get('default_hashtags', []),
ai_prompt=template_data.get('ai_prompt', ''),
ai_tone=template_data.get('ai_tone', ''),
ai_target_audience=template_data.get('ai_target_audience', '')
)
session.add(template)
session.commit()
logger.info(f"Saved content template: {template.name}")
return template
except Exception as e:
session.rollback()
logger.error(f"Error saving content template: {e}")
raise
finally:
session.close()
def get_user_templates(self, user_id: str, category: Optional[ContentCategory] = None) -> List[ContentTemplate]:
"""Get content templates for a user."""
session = self.get_session()
try:
user = session.query(TwitterUser).filter_by(user_id=user_id).first()
if not user:
return []
query = session.query(ContentTemplate).filter(
and_(
ContentTemplate.user_id == user.id,
ContentTemplate.is_active == True
)
)
if category:
query = query.filter_by(category=category)
return query.order_by(desc(ContentTemplate.average_performance)).all()
finally:
session.close()
# --- SETTINGS ---
def save_user_settings(self, user_id: str, settings_data: Dict[str, Any]) -> TwitterSettings:
"""Save user Twitter settings."""
session = self.get_session()
try:
user = session.query(TwitterUser).filter_by(user_id=user_id).first()
if not user:
raise ValueError(f"User {user_id} not found")
# Check if settings already exist
existing_settings = session.query(TwitterSettings).filter_by(user_id=user.id).first()
if existing_settings:
# Update existing settings
for key, value in settings_data.items():
if hasattr(existing_settings, key):
setattr(existing_settings, key, value)
existing_settings.updated_at = datetime.utcnow()
session.commit()
return existing_settings
else:
# Create new settings
settings = TwitterSettings(
user_id=user.id,
**settings_data
)
session.add(settings)
session.commit()
logger.info(f"Saved settings for user: {user.username}")
return settings
except Exception as e:
session.rollback()
logger.error(f"Error saving user settings: {e}")
raise
finally:
session.close()
def get_user_settings(self, user_id: str) -> Optional[TwitterSettings]:
"""Get user Twitter settings."""
session = self.get_session()
try:
user = session.query(TwitterUser).filter_by(user_id=user_id).first()
if not user:
return None
return session.query(TwitterSettings).filter_by(user_id=user.id).first()
finally:
session.close()
# --- UTILITY METHODS ---
def cleanup_old_data(self, days_old: int = 30) -> Dict[str, int]:
"""
Clean up old data to maintain database performance.
Args:
days_old: Number of days old data to keep
Returns:
Dictionary with cleanup statistics
"""
try:
cutoff_date = datetime.utcnow() - timedelta(days=days_old)
with self.get_session() as session:
# Clean up old analytics data
old_analytics = session.query(TwitterAnalytics).filter(
TwitterAnalytics.created_at < cutoff_date
).count()
session.query(TwitterAnalytics).filter(
TwitterAnalytics.created_at < cutoff_date
).delete()
# Clean up old tweet analytics
old_tweet_analytics = session.query(TweetAnalytics).filter(
TweetAnalytics.created_at < cutoff_date
).count()
session.query(TweetAnalytics).filter(
TweetAnalytics.created_at < cutoff_date
).delete()
session.commit()
stats = {
'old_analytics_removed': old_analytics,
'old_tweet_analytics_removed': old_tweet_analytics,
'cutoff_date': cutoff_date.isoformat()
}
logger.info(f"Cleaned up old data: {stats}")
return stats
except Exception as e:
logger.error(f"Error cleaning up old data: {e}")
return {'error': str(e)}
def get_database_stats(self) -> Dict[str, int]:
"""
Get database statistics.
Returns:
Dictionary with database statistics
"""
try:
with self.get_session() as session:
stats = {
'total_users': session.query(TwitterUser).count(),
'total_tweets': session.query(Tweet).count(),
'posted_tweets': session.query(Tweet).filter(
Tweet.status == TweetStatus.POSTED
).count(),
'scheduled_tweets': session.query(ScheduledTweet).filter(
ScheduledTweet.status == TweetStatus.SCHEDULED
).count(),
'total_analytics_records': session.query(TwitterAnalytics).count(),
'total_templates': session.query(ContentTemplate).count()
}
return stats
except Exception as e:
logger.error(f"Error getting database stats: {e}")
return {'error': str(e)}
def close(self):
"""
Close database connections and clean up resources.
"""
try:
if hasattr(self, 'engine') and self.engine:
self.engine.dispose()
logger.info("Database connections closed successfully")
except Exception as e:
logger.error(f"Error closing database connections: {e}")
# Create a global instance for easy access
twitter_db = TwitterDatabaseService()
# Export the service and key functions
__all__ = [
'TwitterDatabaseService',
'twitter_db'
]