kunthawat/moreminimore-marketing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Base code

Browse Source
This commit is contained in:
Kunthawat Greethong
2026-01-08 22:39:53 +07:00
parent 697115c61a
commit c35fa52117
2169 changed files with 626670 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

904
backend/services/content_gap_analyzer/ai_engine_service.py Normal file
View File

@@ -0,0 +1,904 @@
"""
AI Engine Service
Provides AI-powered insights and analysis for content planning.
"""
from typing import Dict, Any, List, Optional
from sqlalchemy.orm import Session
from loguru import logger
from datetime import datetime
import asyncio
import json
from collections import Counter, defaultdict
# Import AI providers
from services.llm_providers.main_text_generation import llm_text_gen
from services.llm_providers.gemini_provider import gemini_structured_json_response
# Import services
from services.ai_service_manager import AIServiceManager
# Import existing modules (will be updated to use FastAPI services)
from services.database import get_db_session
class AIEngineService:
"""AI engine for content planning insights and analysis."""
_instance = None
_initialized = False
def __new__(cls):
"""Implement singleton pattern to prevent multiple initializations."""
if cls._instance is None:
cls._instance = super(AIEngineService, cls).__new__(cls)
return cls._instance
def __init__(self):
"""Initialize the AI engine service (only once)."""
if not self._initialized:
self.ai_service_manager = AIServiceManager()
logger.debug("AIEngineService initialized")
self._initialized = True
async def analyze_content_gaps(self, analysis_summary: Dict[str, Any]) -> Dict[str, Any]:
"""
Analyze content gaps using AI insights.
Args:
analysis_summary: Summary of content analysis
Returns:
AI-powered content gap insights
"""
try:
logger.info("🤖 Generating AI-powered content gap insights using centralized AI service")
# Use the centralized AI service manager for strategic analysis
result = await self.ai_service_manager.generate_content_gap_analysis(analysis_summary)
logger.info("✅ Advanced AI content gap analysis completed")
return result
except Exception as e:
logger.error(f"Error in AI content gap analysis: {str(e)}")
# Return fallback response if AI fails
return {
'strategic_insights': [
{
'type': 'content_strategy',
'insight': 'Focus on educational content to build authority',
'confidence': 0.85,
'priority': 'high',
'estimated_impact': 'Authority building'
}
],
'content_recommendations': [
{
'type': 'content_creation',
'recommendation': 'Create comprehensive guides for high-opportunity keywords',
'priority': 'high',
'estimated_traffic': '5K+ monthly',
'implementation_time': '2-3 weeks'
}
],
'performance_predictions': {
'estimated_traffic_increase': '25%',
'estimated_ranking_improvement': '15 positions',
'estimated_engagement_increase': '30%',
'estimated_conversion_increase': '20%',
'confidence_level': '85%'
},
'risk_assessment': {
'content_quality_risk': 'Low',
'competition_risk': 'Medium',
'implementation_risk': 'Low',
'timeline_risk': 'Medium',
'overall_risk': 'Low'
}
}
async def analyze_market_position(self, market_data: Dict[str, Any]) -> Dict[str, Any]:
"""
Analyze market position using AI insights.
Args:
market_data: Market analysis data
Returns:
AI-powered market position analysis
"""
try:
logger.info("🤖 Generating AI-powered market position analysis using centralized AI service")
# Use the centralized AI service manager for market position analysis
result = await self.ai_service_manager.generate_market_position_analysis(market_data)
logger.info("✅ Advanced AI market position analysis completed")
return result
except Exception as e:
logger.error(f"Error in AI market position analysis: {str(e)}")
# Return fallback response if AI fails
return {
'market_leader': 'competitor1.com',
'content_leader': 'competitor2.com',
'quality_leader': 'competitor3.com',
'market_gaps': [
'Video content',
'Interactive content',
'User-generated content',
'Expert interviews',
'Industry reports'
],
'opportunities': [
'Niche content development',
'Expert interviews',
'Industry reports',
'Case studies',
'Tutorial series'
],
'competitive_advantages': [
'Technical expertise',
'Comprehensive guides',
'Industry insights',
'Expert opinions'
],
'strategic_recommendations': [
{
'type': 'differentiation',
'recommendation': 'Focus on unique content angles',
'priority': 'high',
'estimated_impact': 'Brand differentiation'
},
{
'type': 'quality',
'recommendation': 'Improve content quality and depth',
'priority': 'high',
'estimated_impact': 'Authority building'
},
{
'type': 'innovation',
'recommendation': 'Develop innovative content formats',
'priority': 'medium',
'estimated_impact': 'Engagement improvement'
}
]
}
async def generate_content_recommendations(self, analysis_data: Dict[str, Any]) -> List[Dict[str, Any]]:
"""
Generate AI-powered content recommendations.
Args:
analysis_data: Content analysis data
Returns:
List of AI-generated content recommendations
"""
try:
logger.info("🤖 Generating AI-powered content recommendations")
# Create comprehensive prompt for content recommendations
prompt = f"""
Generate content recommendations based on the following analysis data:
Analysis Data: {json.dumps(analysis_data, indent=2)}
Provide detailed content recommendations including:
1. Content creation opportunities
2. Content optimization suggestions
3. Content series development
4. Content format recommendations
5. Implementation priorities
6. Estimated impact and timeline
Format as structured JSON with detailed recommendations.
"""
# Use structured JSON response for better parsing
response = gemini_structured_json_response(
prompt=prompt,
schema={
"type": "object",
"properties": {
"recommendations": {
"type": "array",
"items": {
"type": "object",
"properties": {
"type": {"type": "string"},
"title": {"type": "string"},
"description": {"type": "string"},
"priority": {"type": "string"},
"estimated_impact": {"type": "string"},
"implementation_time": {"type": "string"},
"ai_confidence": {"type": "number"},
"content_suggestions": {
"type": "array",
"items": {"type": "string"}
}
}
}
}
}
}
)
# Handle response - gemini_structured_json_response returns dict directly
if isinstance(response, dict):
result = response
elif isinstance(response, str):
# If it's a string, try to parse as JSON
try:
result = json.loads(response)
except json.JSONDecodeError as e:
logger.error(f"Failed to parse AI response as JSON: {e}")
raise Exception(f"Invalid AI response format: {str(e)}")
else:
logger.error(f"Unexpected response type from AI service: {type(response)}")
raise Exception(f"Unexpected response type from AI service: {type(response)}")
recommendations = result.get('recommendations', [])
logger.info(f"✅ Generated {len(recommendations)} AI content recommendations")
return recommendations
except Exception as e:
logger.error(f"Error generating AI content recommendations: {str(e)}")
# Return fallback response if AI fails
return [
{
'type': 'content_creation',
'title': 'Create comprehensive guide for target keyword',
'description': 'Develop in-depth guide covering all aspects of the topic',
'priority': 'high',
'estimated_impact': '5K+ monthly traffic',
'implementation_time': '2-3 weeks',
'ai_confidence': 0.92,
'content_suggestions': [
'Step-by-step tutorial',
'Best practices section',
'Common mistakes to avoid',
'Expert tips and insights'
]
},
{
'type': 'content_optimization',
'title': 'Optimize existing content for target keywords',
'description': 'Update current content to improve rankings',
'priority': 'medium',
'estimated_impact': '2K+ monthly traffic',
'implementation_time': '1-2 weeks',
'ai_confidence': 0.88,
'content_suggestions': [
'Add target keywords naturally',
'Improve meta descriptions',
'Enhance internal linking',
'Update outdated information'
]
},
{
'type': 'content_series',
'title': 'Develop content series around main topic',
'description': 'Create interconnected content pieces',
'priority': 'medium',
'estimated_impact': '3K+ monthly traffic',
'implementation_time': '4-6 weeks',
'ai_confidence': 0.85,
'content_suggestions': [
'Part 1: Introduction and basics',
'Part 2: Advanced techniques',
'Part 3: Expert-level insights',
'Part 4: Case studies and examples'
]
}
]
async def predict_content_performance(self, content_data: Dict[str, Any]) -> Dict[str, Any]:
"""
Predict content performance using AI.
Args:
content_data: Content analysis data
Returns:
AI-powered performance predictions
"""
try:
logger.info("🤖 Generating AI-powered performance predictions")
# Create comprehensive prompt for performance prediction
prompt = f"""
Predict content performance based on the following data:
Content Data: {json.dumps(content_data, indent=2)}
Provide detailed performance predictions including:
1. Traffic predictions
2. Engagement predictions
3. Ranking predictions
4. Conversion predictions
5. Risk factors
6. Success factors
Format as structured JSON with confidence levels.
"""
# Use structured JSON response for better parsing
response = gemini_structured_json_response(
prompt=prompt,
schema={
"type": "object",
"properties": {
"traffic_predictions": {
"type": "object",
"properties": {
"estimated_monthly_traffic": {"type": "string"},
"traffic_growth_rate": {"type": "string"},
"peak_traffic_month": {"type": "string"},
"confidence_level": {"type": "string"}
}
},
"engagement_predictions": {
"type": "object",
"properties": {
"estimated_time_on_page": {"type": "string"},
"estimated_bounce_rate": {"type": "string"},
"estimated_social_shares": {"type": "string"},
"estimated_comments": {"type": "string"},
"confidence_level": {"type": "string"}
}
},
"ranking_predictions": {
"type": "object",
"properties": {
"estimated_ranking_position": {"type": "string"},
"estimated_ranking_time": {"type": "string"},
"ranking_confidence": {"type": "string"},
"competition_level": {"type": "string"}
}
},
"conversion_predictions": {
"type": "object",
"properties": {
"estimated_conversion_rate": {"type": "string"},
"estimated_lead_generation": {"type": "string"},
"estimated_revenue_impact": {"type": "string"},
"confidence_level": {"type": "string"}
}
},
"risk_factors": {
"type": "array",
"items": {"type": "string"}
},
"success_factors": {
"type": "array",
"items": {"type": "string"}
}
}
}
)
# Handle response - gemini_structured_json_response returns dict directly
if isinstance(response, dict):
predictions = response
elif isinstance(response, str):
# If it's a string, try to parse as JSON
try:
predictions = json.loads(response)
except json.JSONDecodeError as e:
logger.error(f"Failed to parse AI response as JSON: {e}")
raise Exception(f"Invalid AI response format: {str(e)}")
else:
logger.error(f"Unexpected response type from AI service: {type(response)}")
raise Exception(f"Unexpected response type from AI service: {type(response)}")
logger.info("✅ AI performance predictions completed")
return predictions
except Exception as e:
logger.error(f"Error in AI performance prediction: {str(e)}")
# Return fallback response if AI fails
return {
'traffic_predictions': {
'estimated_monthly_traffic': '5K+',
'traffic_growth_rate': '25%',
'peak_traffic_month': 'Q4',
'confidence_level': '85%'
},
'engagement_predictions': {
'estimated_time_on_page': '3-5 minutes',
'estimated_bounce_rate': '35%',
'estimated_social_shares': '50+',
'estimated_comments': '15+',
'confidence_level': '80%'
},
'ranking_predictions': {
'estimated_ranking_position': 'Top 10',
'estimated_ranking_time': '2-3 months',
'ranking_confidence': '75%',
'competition_level': 'Medium'
},
'conversion_predictions': {
'estimated_conversion_rate': '3-5%',
'estimated_lead_generation': '100+ monthly',
'estimated_revenue_impact': '$10K+ monthly',
'confidence_level': '70%'
},
'risk_factors': [
'High competition for target keywords',
'Seasonal content performance variations',
'Content quality requirements',
'Implementation timeline constraints'
],
'success_factors': [
'Comprehensive content coverage',
'Expert-level insights',
'Engaging content format',
'Strong internal linking',
'Regular content updates'
]
}
async def analyze_competitive_intelligence(self, competitor_data: Dict[str, Any]) -> Dict[str, Any]:
"""
Analyze competitive intelligence using AI.
Args:
competitor_data: Competitor analysis data
Returns:
AI-powered competitive intelligence
"""
try:
logger.info("🤖 Generating AI-powered competitive intelligence")
# Create comprehensive prompt for competitive intelligence
prompt = f"""
Analyze competitive intelligence based on the following competitor data:
Competitor Data: {json.dumps(competitor_data, indent=2)}
Provide comprehensive competitive intelligence including:
1. Market analysis
2. Content strategy insights
3. Competitive advantages
4. Threat analysis
5. Opportunity analysis
Format as structured JSON with detailed analysis.
"""
# Use structured JSON response for better parsing
response = gemini_structured_json_response(
prompt=prompt,
schema={
"type": "object",
"properties": {
"market_analysis": {
"type": "object",
"properties": {
"market_leader": {"type": "string"},
"content_leader": {"type": "string"},
"innovation_leader": {"type": "string"},
"market_gaps": {
"type": "array",
"items": {"type": "string"}
}
}
},
"content_strategy_insights": {
"type": "array",
"items": {
"type": "object",
"properties": {
"insight": {"type": "string"},
"opportunity": {"type": "string"},
"priority": {"type": "string"},
"estimated_impact": {"type": "string"}
}
}
},
"competitive_advantages": {
"type": "array",
"items": {"type": "string"}
},
"threat_analysis": {
"type": "array",
"items": {
"type": "object",
"properties": {
"threat": {"type": "string"},
"risk_level": {"type": "string"},
"mitigation": {"type": "string"}
}
}
},
"opportunity_analysis": {
"type": "array",
"items": {
"type": "object",
"properties": {
"opportunity": {"type": "string"},
"market_gap": {"type": "string"},
"estimated_impact": {"type": "string"},
"implementation_time": {"type": "string"}
}
}
}
}
}
)
# Parse and return the AI response
# Handle response - gemini_structured_json_response returns dict directly
if isinstance(response, dict):
competitive_intelligence = response
elif isinstance(response, str):
# If it's a string, try to parse as JSON
try:
competitive_intelligence = json.loads(response)
except json.JSONDecodeError as e:
logger.error(f"Failed to parse AI response as JSON: {e}")
raise Exception(f"Invalid AI response format: {str(e)}")
else:
logger.error(f"Unexpected response type from AI service: {type(response)}")
raise Exception(f"Unexpected response type from AI service: {type(response)}")
logger.info("✅ AI competitive intelligence completed")
return competitive_intelligence
except Exception as e:
logger.error(f"Error in AI competitive intelligence: {str(e)}")
# Return fallback response if AI fails
return {
'market_analysis': {
'market_leader': 'competitor1.com',
'content_leader': 'competitor2.com',
'innovation_leader': 'competitor3.com',
'market_gaps': [
'Video tutorials',
'Interactive content',
'Expert interviews',
'Industry reports'
]
},
'content_strategy_insights': [
{
'insight': 'Competitors focus heavily on educational content',
'opportunity': 'Develop unique content angles',
'priority': 'high',
'estimated_impact': 'Differentiation'
},
{
'insight': 'Limited video content in the market',
'opportunity': 'Create video tutorials and guides',
'priority': 'medium',
'estimated_impact': 'Engagement improvement'
},
{
'insight': 'High demand for expert insights',
'opportunity': 'Develop expert interview series',
'priority': 'high',
'estimated_impact': 'Authority building'
}
],
'competitive_advantages': [
'Technical expertise',
'Comprehensive content coverage',
'Industry insights',
'Expert opinions',
'Practical examples'
],
'threat_analysis': [
{
'threat': 'Competitor content quality improvement',
'risk_level': 'Medium',
'mitigation': 'Focus on unique value propositions'
},
{
'threat': 'New competitors entering market',
'risk_level': 'Low',
'mitigation': 'Build strong brand authority'
},
{
'threat': 'Content saturation in key topics',
'risk_level': 'High',
'mitigation': 'Develop niche content areas'
}
],
'opportunity_analysis': [
{
'opportunity': 'Video content development',
'market_gap': 'Limited video tutorials',
'estimated_impact': 'High engagement',
'implementation_time': '3-6 months'
},
{
'opportunity': 'Expert interview series',
'market_gap': 'Lack of expert insights',
'estimated_impact': 'Authority building',
'implementation_time': '2-4 months'
},
{
'opportunity': 'Interactive content',
'market_gap': 'No interactive elements',
'estimated_impact': 'User engagement',
'implementation_time': '1-3 months'
}
]
}
async def generate_strategic_insights(self, analysis_data: Dict[str, Any]) -> List[Dict[str, Any]]:
"""
Generate strategic insights using AI.
Args:
analysis_data: Analysis data
Returns:
List of AI-generated strategic insights
"""
try:
logger.info("🤖 Generating AI-powered strategic insights")
# Create comprehensive prompt for strategic insights
prompt = f"""
Generate strategic insights based on the following analysis data:
Analysis Data: {json.dumps(analysis_data, indent=2)}
Provide strategic insights covering:
1. Content strategy recommendations
2. Competitive positioning advice
3. Content optimization suggestions
4. Innovation opportunities
5. Risk mitigation strategies
Format as structured JSON with detailed insights.
"""
# Use structured JSON response for better parsing
response = gemini_structured_json_response(
prompt=prompt,
schema={
"type": "object",
"properties": {
"strategic_insights": {
"type": "array",
"items": {
"type": "object",
"properties": {
"type": {"type": "string"},
"insight": {"type": "string"},
"reasoning": {"type": "string"},
"priority": {"type": "string"},
"estimated_impact": {"type": "string"},
"implementation_time": {"type": "string"}
}
}
}
}
}
)
# Handle response - gemini_structured_json_response returns dict directly
if isinstance(response, dict):
result = response
elif isinstance(response, str):
# If it's a string, try to parse as JSON
try:
result = json.loads(response)
except json.JSONDecodeError as e:
logger.error(f"Failed to parse AI response as JSON: {e}")
raise Exception(f"Invalid AI response format: {str(e)}")
else:
logger.error(f"Unexpected response type from AI service: {type(response)}")
raise Exception(f"Unexpected response type from AI service: {type(response)}")
strategic_insights = result.get('strategic_insights', [])
logger.info(f"✅ Generated {len(strategic_insights)} AI strategic insights")
return strategic_insights
except Exception as e:
logger.error(f"Error generating AI strategic insights: {str(e)}")
# Return fallback response if AI fails
return [
{
'type': 'content_strategy',
'insight': 'Focus on educational content to build authority and trust',
'reasoning': 'High informational search intent indicates need for educational content',
'priority': 'high',
'estimated_impact': 'Authority building',
'implementation_time': '3-6 months'
},
{
'type': 'competitive_positioning',
'insight': 'Differentiate through unique content angles and expert insights',
'reasoning': 'Competitors lack expert-level content and unique perspectives',
'priority': 'high',
'estimated_impact': 'Brand differentiation',
'implementation_time': '2-4 months'
},
{
'type': 'content_optimization',
'insight': 'Optimize existing content for target keywords and user intent',
'reasoning': 'Current content not fully optimized for search and user needs',
'priority': 'medium',
'estimated_impact': 'Improved rankings',
'implementation_time': '1-2 months'
},
{
'type': 'content_innovation',
'insight': 'Develop video and interactive content to stand out',
'reasoning': 'Market lacks engaging multimedia content',
'priority': 'medium',
'estimated_impact': 'Engagement improvement',
'implementation_time': '3-6 months'
},
{
'type': 'content_series',
'insight': 'Create comprehensive content series around main topics',
'reasoning': 'Series content performs better and builds authority',
'priority': 'medium',
'estimated_impact': 'User retention',
'implementation_time': '4-8 weeks'
}
]
async def analyze_content_quality(self, content_data: Dict[str, Any]) -> Dict[str, Any]:
"""
Analyze content quality and provide improvement suggestions.
Args:
content_data: Content data to analyze
Returns:
Content quality analysis
"""
try:
logger.info("Analyzing content quality using AI")
# Create comprehensive prompt for content quality analysis
prompt = f"""
Analyze the quality of the following content and provide improvement suggestions:
Content Data: {json.dumps(content_data, indent=2)}
Provide comprehensive content quality analysis including:
1. Overall quality score
2. Readability assessment
3. SEO optimization analysis
4. Engagement potential evaluation
5. Improvement suggestions
Format as structured JSON with detailed analysis.
"""
# Use structured JSON response for better parsing
response = gemini_structured_json_response(
prompt=prompt,
schema={
"type": "object",
"properties": {
"overall_score": {"type": "number"},
"readability_score": {"type": "number"},
"seo_score": {"type": "number"},
"engagement_potential": {"type": "string"},
"improvement_suggestions": {
"type": "array",
"items": {"type": "string"}
},
"timestamp": {"type": "string"}
}
}
)
# Handle response - gemini_structured_json_response returns dict directly
if isinstance(response, dict):
quality_analysis = response
elif isinstance(response, str):
# If it's a string, try to parse as JSON
try:
quality_analysis = json.loads(response)
except json.JSONDecodeError as e:
logger.error(f"Failed to parse AI response as JSON: {e}")
raise Exception(f"Invalid AI response format: {str(e)}")
else:
logger.error(f"Unexpected response type from AI service: {type(response)}")
raise Exception(f"Unexpected response type from AI service: {type(response)}")
logger.info("✅ AI content quality analysis completed")
return quality_analysis
except Exception as e:
logger.error(f"Error analyzing content quality: {str(e)}")
# Return fallback response if AI fails
return {
'overall_score': 8.5,
'readability_score': 9.2,
'seo_score': 7.8,
'engagement_potential': 'High',
'improvement_suggestions': [
'Add more subheadings for better structure',
'Include more relevant keywords naturally',
'Add call-to-action elements',
'Optimize for mobile reading'
],
'timestamp': datetime.utcnow().isoformat()
}
async def health_check(self) -> Dict[str, Any]:
"""
Health check for the AI engine service.
Returns:
Health status information
"""
try:
logger.info("Performing health check for AIEngineService")
# Test AI functionality with a simple prompt
test_prompt = "Hello, this is a health check test."
try:
test_response = llm_text_gen(test_prompt)
ai_status = "operational" if test_response else "degraded"
except Exception as e:
ai_status = "error"
logger.warning(f"AI health check failed: {str(e)}")
health_status = {
'service': 'AIEngineService',
'status': 'healthy',
'capabilities': {
'content_analysis': 'operational',
'strategy_generation': 'operational',
'recommendation_engine': 'operational',
'quality_assessment': 'operational',
'ai_integration': ai_status
},
'timestamp': datetime.utcnow().isoformat()
}
logger.info("AIEngineService health check passed")
return health_status
except Exception as e:
logger.error(f"AIEngineService health check failed: {str(e)}")
return {
'service': 'AIEngineService',
'status': 'unhealthy',
'error': str(e),
'timestamp': datetime.utcnow().isoformat()
}
async def get_ai_summary(self, analysis_id: str) -> Dict[str, Any]:
"""
Get summary of AI analysis.
Args:
analysis_id: Analysis identifier
Returns:
AI analysis summary
"""
try:
logger.info(f"Getting AI analysis summary for {analysis_id}")
# TODO: Retrieve analysis from database
# This will be implemented when database integration is complete
summary = {
'analysis_id': analysis_id,
'status': 'completed',
'timestamp': datetime.utcnow().isoformat(),
'summary': {
'ai_insights_generated': 15,
'strategic_recommendations': 8,
'performance_predictions': 'Completed',
'competitive_intelligence': 'Analyzed',
'content_quality_score': 8.5,
'estimated_impact': 'High'
}
}
return summary
except Exception as e:
logger.error(f"Error getting AI summary: {str(e)}")
return {}

1243
backend/services/content_gap_analyzer/competitor_analyzer.py Normal file
View File

File diff suppressed because it is too large Load Diff

853
backend/services/content_gap_analyzer/content_gap_analyzer.py Normal file
View File

@@ -0,0 +1,853 @@
"""
Content Gap Analyzer Service
Converted from enhanced_analyzer.py for FastAPI integration.
"""
from typing import Dict, Any, List, Optional
from sqlalchemy.orm import Session
from loguru import logger
from datetime import datetime
import asyncio
import json
import pandas as pd
import advertools as adv
import tempfile
import os
from urllib.parse import urlparse
from collections import Counter, defaultdict
# Import existing modules (will be updated to use FastAPI services)
from services.database import get_db_session
from .ai_engine_service import AIEngineService
from .competitor_analyzer import CompetitorAnalyzer
from .keyword_researcher import KeywordResearcher
class ContentGapAnalyzer:
"""Enhanced content gap analyzer with advertools integration and AI insights."""
def __init__(self):
"""Initialize the enhanced analyzer."""
self.ai_engine = AIEngineService()
self.competitor_analyzer = CompetitorAnalyzer()
self.keyword_researcher = KeywordResearcher()
# Temporary directories for crawl data
self.temp_dir = tempfile.mkdtemp()
logger.info("ContentGapAnalyzer initialized")
async def analyze_comprehensive_gap(self, target_url: str, competitor_urls: List[str],
target_keywords: List[str], industry: str = "general") -> Dict[str, Any]:
"""
Perform comprehensive content gap analysis.
Args:
target_url: Your website URL
competitor_urls: List of competitor URLs (max 5 for performance)
target_keywords: List of primary keywords to analyze
industry: Industry category for context
Returns:
Comprehensive analysis results
"""
try:
logger.info(f"🚀 Starting Enhanced Content Gap Analysis for {target_url}")
# Initialize results structure
results = {
'analysis_timestamp': datetime.utcnow().isoformat(),
'target_url': target_url,
'competitor_urls': competitor_urls[:5], # Limit to 5 competitors
'target_keywords': target_keywords,
'industry': industry,
'serp_analysis': {},
'keyword_expansion': {},
'competitor_content': {},
'content_themes': {},
'gap_analysis': {},
'ai_insights': {},
'recommendations': []
}
# Phase 1: SERP Analysis using adv.serp_goog
logger.info("🔍 Starting SERP Analysis")
serp_results = await self._analyze_serp_landscape(target_keywords, competitor_urls)
results['serp_analysis'] = serp_results
logger.info(f"✅ Analyzed {len(target_keywords)} keywords across SERPs")
# Phase 2: Keyword Expansion using adv.kw_generate
logger.info("🎯 Starting Keyword Research Expansion")
expanded_keywords = await self._expand_keyword_research(target_keywords, industry)
results['keyword_expansion'] = expanded_keywords
logger.info(f"✅ Generated {len(expanded_keywords.get('expanded_keywords', []))} additional keywords")
# Phase 3: Deep Competitor Analysis using adv.crawl
logger.info("🕷️ Starting Deep Competitor Content Analysis")
competitor_content = await self._analyze_competitor_content_deep(competitor_urls)
results['competitor_content'] = competitor_content
logger.info(f"✅ Crawled and analyzed {len(competitor_urls)} competitor websites")
# Phase 4: Content Theme Analysis using adv.word_frequency
logger.info("📊 Starting Content Theme & Gap Identification")
content_themes = await self._analyze_content_themes(results['competitor_content'])
results['content_themes'] = content_themes
logger.info("✅ Identified content themes and topic clusters")
# Phase 5: AI-Powered Insights
logger.info("🤖 Generating AI-powered insights")
ai_insights = await self._generate_ai_insights(results)
results['ai_insights'] = ai_insights
logger.info("✅ Generated comprehensive AI insights")
# Phase 6: Gap Analysis
logger.info("🔍 Performing comprehensive gap analysis")
gap_analysis = await self._perform_gap_analysis(results)
results['gap_analysis'] = gap_analysis
logger.info("✅ Completed gap analysis")
# Phase 7: Strategic Recommendations
logger.info("🎯 Generating strategic recommendations")
recommendations = await self._generate_strategic_recommendations(results)
results['recommendations'] = recommendations
logger.info("✅ Generated strategic recommendations")
logger.info(f"🎉 Comprehensive content gap analysis completed for {target_url}")
return results
except Exception as e:
error_msg = f"Error in comprehensive gap analysis: {str(e)}"
logger.error(error_msg, exc_info=True)
return {'error': error_msg}
async def _analyze_serp_landscape(self, keywords: List[str], competitor_urls: List[str]) -> Dict[str, Any]:
"""
Analyze SERP landscape using adv.serp_goog.
Args:
keywords: List of keywords to analyze
competitor_urls: List of competitor URLs
Returns:
SERP analysis results
"""
try:
logger.info(f"Analyzing SERP landscape for {len(keywords)} keywords")
serp_results = {
'keyword_rankings': {},
'competitor_presence': {},
'serp_features': {},
'ranking_opportunities': []
}
# Note: adv.serp_goog requires API key setup
# For demo purposes, we'll simulate SERP analysis with structured data
for keyword in keywords[:10]: # Limit to prevent API overuse
try:
# In production, use: serp_data = adv.serp_goog(q=keyword, cx='your_cx', key='your_key')
# For now, we'll create structured placeholder data that mimics real SERP analysis
# Simulate SERP data structure
serp_data = {
'keyword': keyword,
'search_volume': f"{1000 + hash(keyword) % 50000}",
'difficulty': ['Low', 'Medium', 'High'][hash(keyword) % 3],
'competition': ['Low', 'Medium', 'High'][hash(keyword) % 3],
'serp_features': ['featured_snippet', 'people_also_ask', 'related_searches'],
'top_10_domains': [urlparse(url).netloc for url in competitor_urls[:5]],
'competitor_positions': {
urlparse(url).netloc: f"Position {i+3}" for i, url in enumerate(competitor_urls[:5])
}
}
serp_results['keyword_rankings'][keyword] = serp_data
# Identify ranking opportunities
target_domain = urlparse(competitor_urls[0] if competitor_urls else "").netloc
if target_domain not in serp_data.get('competitor_positions', {}):
serp_results['ranking_opportunities'].append({
'keyword': keyword,
'opportunity': 'Not ranking in top 10',
'serp_features': serp_data.get('serp_features', []),
'estimated_traffic': serp_data.get('search_volume', 'Unknown'),
'competition_level': serp_data.get('difficulty', 'Unknown')
})
logger.info(f"• Analyzed keyword: '{keyword}'")
except Exception as e:
logger.warning(f"Could not analyze SERP for '{keyword}': {str(e)}")
continue
# Analyze competitor SERP presence
domain_counts = Counter()
for keyword_data in serp_results['keyword_rankings'].values():
for domain in keyword_data.get('top_10_domains', []):
domain_counts[domain] += 1
serp_results['competitor_presence'] = dict(domain_counts.most_common(10))
logger.info(f"SERP analysis completed for {len(keywords)} keywords")
return serp_results
except Exception as e:
logger.error(f"Error in SERP analysis: {str(e)}")
return {}
async def _expand_keyword_research(self, seed_keywords: List[str], industry: str) -> Dict[str, Any]:
"""
Expand keyword research using adv.kw_generate.
Args:
seed_keywords: Initial keywords to expand from
industry: Industry category
Returns:
Expanded keyword research results
"""
try:
logger.info(f"Expanding keyword research for {industry} industry")
expanded_results = {
'seed_keywords': seed_keywords,
'expanded_keywords': [],
'keyword_categories': {},
'search_intent_analysis': {},
'long_tail_opportunities': []
}
# Use adv.kw_generate for keyword expansion
all_expanded = []
for seed_keyword in seed_keywords[:5]: # Limit to prevent overload
try:
# Generate keyword variations using advertools
# In production, use actual adv.kw_generate
# For demo, we'll simulate the expansion
# Simulate broad keyword generation
broad_keywords = [
f"{seed_keyword} guide",
f"best {seed_keyword}",
f"how to {seed_keyword}",
f"{seed_keyword} tips",
f"{seed_keyword} tutorial",
f"{seed_keyword} examples",
f"{seed_keyword} vs",
f"{seed_keyword} review",
f"{seed_keyword} comparison"
]
# Simulate phrase match keywords
phrase_keywords = [
f"{industry} {seed_keyword}",
f"{seed_keyword} {industry} strategy",
f"{seed_keyword} {industry} analysis",
f"{seed_keyword} {industry} optimization",
f"{seed_keyword} {industry} techniques"
]
all_expanded.extend(broad_keywords)
all_expanded.extend(phrase_keywords)
logger.info(f"• Generated variations for: '{seed_keyword}'")
except Exception as e:
logger.warning(f"Could not expand keyword '{seed_keyword}': {str(e)}")
continue
# Remove duplicates and clean
expanded_results['expanded_keywords'] = list(set(all_expanded))
# Categorize keywords by intent
intent_categories = {
'informational': [],
'commercial': [],
'navigational': [],
'transactional': []
}
for keyword in expanded_results['expanded_keywords']:
keyword_lower = keyword.lower()
if any(word in keyword_lower for word in ['how', 'what', 'why', 'guide', 'tips', 'tutorial']):
intent_categories['informational'].append(keyword)
elif any(word in keyword_lower for word in ['best', 'top', 'review', 'comparison', 'vs']):
intent_categories['commercial'].append(keyword)
elif any(word in keyword_lower for word in ['buy', 'purchase', 'price', 'cost']):
intent_categories['transactional'].append(keyword)
else:
intent_categories['navigational'].append(keyword)
expanded_results['keyword_categories'] = intent_categories
# Identify long-tail opportunities
long_tail = [kw for kw in expanded_results['expanded_keywords'] if len(kw.split()) >= 3]
expanded_results['long_tail_opportunities'] = long_tail[:20] # Top 20 long-tail
logger.info(f"Keyword expansion completed: {len(expanded_results['expanded_keywords'])} keywords generated")
return expanded_results
except Exception as e:
logger.error(f"Error in keyword expansion: {str(e)}")
return {}
async def _analyze_competitor_content_deep(self, competitor_urls: List[str]) -> Dict[str, Any]:
"""
Deep competitor content analysis using adv.crawl.
Args:
competitor_urls: List of competitor URLs to analyze
Returns:
Deep competitor analysis results
"""
try:
logger.info(f"Starting deep competitor analysis for {len(competitor_urls)} competitors")
competitor_analysis = {
'crawl_results': {},
'content_structure': {},
'page_analysis': {},
'technical_insights': {}
}
for i, url in enumerate(competitor_urls[:3]): # Limit to 3 for performance
try:
domain = urlparse(url).netloc
logger.info(f"🔍 Analyzing competitor {i+1}: {domain}")
# Create temporary file for crawl results
crawl_file = os.path.join(self.temp_dir, f"crawl_{domain.replace('.', '_')}.jl")
# Use adv.crawl for comprehensive analysis
# Note: This is a simplified crawl - in production, customize settings
try:
adv.crawl(
url_list=[url],
output_file=crawl_file,
follow_links=True,
custom_settings={
'DEPTH_LIMIT': 2, # Crawl 2 levels deep
'CLOSESPIDER_PAGECOUNT': 50, # Limit pages
'DOWNLOAD_DELAY': 1, # Be respectful
}
)
# Read and analyze crawl results
if os.path.exists(crawl_file):
crawl_df = pd.read_json(crawl_file, lines=True)
competitor_analysis['crawl_results'][domain] = {
'total_pages': len(crawl_df),
'status_codes': crawl_df['status'].value_counts().to_dict() if 'status' in crawl_df.columns else {},
'page_types': self._categorize_pages(crawl_df),
'content_length_stats': {
'mean': crawl_df['size'].mean() if 'size' in crawl_df.columns else 0,
'median': crawl_df['size'].median() if 'size' in crawl_df.columns else 0
}
}
# Analyze content structure
competitor_analysis['content_structure'][domain] = self._analyze_content_structure(crawl_df)
logger.info(f"✅ Crawled {len(crawl_df)} pages from {domain}")
else:
logger.warning(f"⚠️ No crawl data available for {domain}")
except Exception as crawl_error:
logger.warning(f"Could not crawl {url}: {str(crawl_error)}")
# Fallback to simulated data
competitor_analysis['crawl_results'][domain] = {
'total_pages': 150,
'status_codes': {'200': 150},
'page_types': {
'blog_posts': 80,
'product_pages': 30,
'landing_pages': 20,
'guides': 20
},
'content_length_stats': {
'mean': 2500,
'median': 2200
}
}
except Exception as e:
logger.warning(f"Could not analyze {url}: {str(e)}")
continue
# Analyze content themes across competitors
all_topics = []
for analysis in competitor_analysis['crawl_results'].values():
# Extract topics from page types
page_types = analysis.get('page_types', {})
if page_types.get('blog_posts', 0) > 0:
all_topics.extend(['Industry trends', 'Best practices', 'Case studies'])
if page_types.get('guides', 0) > 0:
all_topics.extend(['Tutorials', 'How-to guides', 'Expert insights'])
topic_frequency = Counter(all_topics)
dominant_themes = topic_frequency.most_common(10)
competitor_analysis['dominant_themes'] = [theme for theme, count in dominant_themes]
competitor_analysis['theme_frequency'] = dict(dominant_themes)
competitor_analysis['content_gaps'] = [
'Video tutorials',
'Interactive content',
'User-generated content',
'Expert interviews',
'Industry reports'
]
competitor_analysis['competitive_advantages'] = [
'Technical expertise',
'Comprehensive guides',
'Industry insights',
'Expert opinions'
]
logger.info(f"Deep competitor analysis completed for {len(competitor_urls)} competitors")
return competitor_analysis
except Exception as e:
logger.error(f"Error in competitor analysis: {str(e)}")
return {}
async def _analyze_content_themes(self, competitor_content: Dict[str, Any]) -> Dict[str, Any]:
"""
Analyze content themes using adv.word_frequency.
Args:
competitor_content: Competitor content analysis results
Returns:
Content theme analysis results
"""
try:
logger.info("Analyzing content themes and topic clusters")
theme_analysis = {
'dominant_themes': {},
'content_clusters': {},
'topic_gaps': [],
'content_opportunities': []
}
all_content_text = ""
# Extract content from crawl results
for domain, crawl_data in competitor_content.get('crawl_results', {}).items():
try:
# In a real implementation, you'd extract text content from crawled pages
# For now, we'll simulate content analysis based on page types
page_types = crawl_data.get('page_types', {})
if page_types.get('blog_posts', 0) > 0:
all_content_text += " content marketing seo optimization digital strategy blog posts articles tutorials guides"
if page_types.get('product_pages', 0) > 0:
all_content_text += " product features benefits comparison reviews testimonials"
if page_types.get('guides', 0) > 0:
all_content_text += " how-to step-by-step instructions best practices tips tricks"
# Add domain-specific content
all_content_text += f" {domain} website analysis competitor research keyword targeting"
except Exception as e:
continue
if all_content_text.strip():
# Use adv.word_frequency for theme analysis
try:
word_freq = adv.word_frequency(
text_list=[all_content_text],
phrase_len=2, # Analyze 2-word phrases
rm_words=['the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by']
)
# Process word frequency results
if not word_freq.empty:
top_themes = word_freq.head(20)
theme_analysis['dominant_themes'] = top_themes.to_dict('records')
# Categorize themes into clusters
theme_analysis['content_clusters'] = self._cluster_themes(top_themes)
except Exception as freq_error:
logger.warning(f"Could not perform word frequency analysis: {str(freq_error)}")
# Fallback to simulated themes
theme_analysis['dominant_themes'] = [
{'word': 'content marketing', 'freq': 45},
{'word': 'seo optimization', 'freq': 38},
{'word': 'digital strategy', 'freq': 32},
{'word': 'best practices', 'freq': 28},
{'word': 'industry insights', 'freq': 25}
]
theme_analysis['content_clusters'] = {
'technical_seo': ['seo optimization', 'keyword targeting'],
'content_marketing': ['content marketing', 'blog posts'],
'business_strategy': ['digital strategy', 'industry insights'],
'user_experience': ['best practices', 'tutorials']
}
logger.info("✅ Identified dominant content themes")
return theme_analysis
except Exception as e:
logger.error(f"Error in content theme analysis: {str(e)}")
return {}
async def _generate_ai_insights(self, analysis_results: Dict[str, Any]) -> Dict[str, Any]:
"""
Generate AI-powered insights using advanced AI analysis.
Args:
analysis_results: Complete analysis results
Returns:
AI-generated insights
"""
try:
logger.info("🤖 Generating AI-powered insights")
# Prepare analysis summary for AI
analysis_summary = {
'target_url': analysis_results.get('target_url', ''),
'industry': analysis_results.get('industry', ''),
'serp_opportunities': len(analysis_results.get('serp_analysis', {}).get('ranking_opportunities', [])),
'expanded_keywords_count': len(analysis_results.get('keyword_expansion', {}).get('expanded_keywords', [])),
'competitors_analyzed': len(analysis_results.get('competitor_urls', [])),
'dominant_themes': analysis_results.get('content_themes', {}).get('dominant_themes', [])[:10]
}
# Generate comprehensive AI insights using AI engine
ai_insights = await self.ai_engine.analyze_content_gaps(analysis_summary)
if ai_insights:
logger.info("✅ Generated comprehensive AI insights")
return ai_insights
else:
logger.warning("⚠️ Could not generate AI insights")
return {}
except Exception as e:
logger.error(f"Error generating AI insights: {str(e)}")
return {}
async def _perform_gap_analysis(self, analysis_results: Dict[str, Any]) -> Dict[str, Any]:
"""
Perform comprehensive gap analysis.
Args:
analysis_results: Complete analysis results
Returns:
Gap analysis results
"""
try:
logger.info("🔍 Performing comprehensive gap analysis")
# Extract key data for gap analysis
serp_opportunities = analysis_results.get('serp_analysis', {}).get('ranking_opportunities', [])
missing_themes = analysis_results.get('content_themes', {}).get('missing_themes', [])
competitor_gaps = analysis_results.get('competitor_content', {}).get('content_gaps', [])
# Identify content gaps
content_gaps = []
# SERP-based gaps
for opportunity in serp_opportunities:
content_gaps.append({
'type': 'keyword_opportunity',
'title': f"Create content for '{opportunity['keyword']}'",
'description': f"Target keyword with {opportunity.get('estimated_traffic', 'Unknown')} monthly traffic",
'priority': 'high' if opportunity.get('opportunity_score', 0) > 7.5 else 'medium',
'estimated_impact': opportunity.get('estimated_traffic', 'Unknown'),
'implementation_time': '2-3 weeks'
})
# Theme-based gaps
for theme in missing_themes:
content_gaps.append({
'type': 'content_theme',
'title': f"Develop {theme.replace('_', ' ').title()} content",
'description': f"Missing content theme with high engagement potential",
'priority': 'medium',
'estimated_impact': 'High engagement',
'implementation_time': '3-4 weeks'
})
# Competitor-based gaps
for gap in competitor_gaps:
content_gaps.append({
'type': 'content_format',
'title': f"Create {gap}",
'description': f"Content format missing from your strategy",
'priority': 'medium',
'estimated_impact': 'Competitive advantage',
'implementation_time': '2-4 weeks'
})
# Calculate gap statistics
gap_stats = {
'total_gaps': len(content_gaps),
'high_priority': len([gap for gap in content_gaps if gap['priority'] == 'high']),
'medium_priority': len([gap for gap in content_gaps if gap['priority'] == 'medium']),
'keyword_opportunities': len([gap for gap in content_gaps if gap['type'] == 'keyword_opportunity']),
'theme_gaps': len([gap for gap in content_gaps if gap['type'] == 'content_theme']),
'format_gaps': len([gap for gap in content_gaps if gap['type'] == 'content_format'])
}
gap_analysis = {
'content_gaps': content_gaps,
'gap_statistics': gap_stats,
'priority_recommendations': sorted(content_gaps, key=lambda x: x['priority'] == 'high', reverse=True)[:5],
'implementation_timeline': {
'immediate': [gap for gap in content_gaps if gap['priority'] == 'high'][:3],
'short_term': [gap for gap in content_gaps if gap['priority'] == 'medium'][:5],
'long_term': [gap for gap in content_gaps if gap['priority'] == 'medium'][5:10]
}
}
logger.info(f"Gap analysis completed: {len(content_gaps)} gaps identified")
return gap_analysis
except Exception as e:
logger.error(f"Error in gap analysis: {str(e)}")
return {}
async def _generate_strategic_recommendations(self, analysis_results: Dict[str, Any]) -> List[Dict[str, Any]]:
"""
Generate strategic recommendations based on analysis results.
Args:
analysis_results: Complete analysis results
Returns:
List of strategic recommendations
"""
try:
logger.info("🎯 Generating strategic recommendations")
recommendations = []
# Keyword-based recommendations
serp_opportunities = analysis_results.get('serp_analysis', {}).get('ranking_opportunities', [])
for opportunity in serp_opportunities[:3]: # Top 3 opportunities
recommendations.append({
'type': 'keyword_optimization',
'title': f"Optimize for '{opportunity['keyword']}'",
'description': f"High-traffic keyword with {opportunity.get('estimated_traffic', 'Unknown')} monthly searches",
'priority': 'high',
'estimated_impact': opportunity.get('estimated_traffic', 'Unknown'),
'implementation_steps': [
f"Create comprehensive content targeting '{opportunity['keyword']}'",
"Optimize on-page SEO elements",
"Build quality backlinks",
"Monitor ranking progress"
]
})
# Content theme recommendations
dominant_themes = analysis_results.get('content_themes', {}).get('dominant_themes', [])
for theme in dominant_themes[:3]: # Top 3 themes
recommendations.append({
'type': 'content_theme',
'title': f"Develop {theme.get('word', 'content theme')} content",
'description': f"High-frequency theme with {theme.get('freq', 0)} mentions across competitors",
'priority': 'medium',
'estimated_impact': 'Increased authority',
'implementation_steps': [
f"Create content series around {theme.get('word', 'theme')}",
"Develop comprehensive guides",
"Create supporting content",
"Promote across channels"
]
})
# Competitive advantage recommendations
competitive_advantages = analysis_results.get('competitor_content', {}).get('competitive_advantages', [])
for advantage in competitive_advantages[:2]: # Top 2 advantages
recommendations.append({
'type': 'competitive_advantage',
'title': f"Develop {advantage}",
'description': f"Competitive advantage identified in analysis",
'priority': 'medium',
'estimated_impact': 'Market differentiation',
'implementation_steps': [
f"Research {advantage} best practices",
"Develop unique approach",
"Create supporting content",
"Promote expertise"
]
})
# Technical SEO recommendations
recommendations.append({
'type': 'technical_seo',
'title': "Improve technical SEO foundation",
'description': "Technical optimization for better search visibility",
'priority': 'high',
'estimated_impact': 'Improved rankings',
'implementation_steps': [
"Audit website technical SEO",
"Fix crawlability issues",
"Optimize page speed",
"Implement structured data"
]
})
# Content strategy recommendations
recommendations.append({
'type': 'content_strategy',
'title': "Develop comprehensive content strategy",
'description': "Strategic content planning for long-term success",
'priority': 'high',
'estimated_impact': 'Sustainable growth',
'implementation_steps': [
"Define content pillars",
"Create editorial calendar",
"Establish content guidelines",
"Set up measurement framework"
]
})
logger.info(f"Strategic recommendations generated: {len(recommendations)} recommendations")
return recommendations
except Exception as e:
logger.error(f"Error generating strategic recommendations: {str(e)}")
return []
def _categorize_pages(self, crawl_df: pd.DataFrame) -> Dict[str, int]:
"""Categorize crawled pages by type."""
page_categories = {
'blog_posts': 0,
'product_pages': 0,
'category_pages': 0,
'landing_pages': 0,
'other': 0
}
if 'url' in crawl_df.columns:
for url in crawl_df['url']:
url_lower = url.lower()
if any(indicator in url_lower for indicator in ['/blog/', '/post/', '/article/', '/news/']):
page_categories['blog_posts'] += 1
elif any(indicator in url_lower for indicator in ['/product/', '/item/', '/shop/']):
page_categories['product_pages'] += 1
elif any(indicator in url_lower for indicator in ['/category/', '/collection/', '/browse/']):
page_categories['category_pages'] += 1
elif any(indicator in url_lower for indicator in ['/landing/', '/promo/', '/campaign/']):
page_categories['landing_pages'] += 1
else:
page_categories['other'] += 1
return page_categories
def _analyze_content_structure(self, crawl_df: pd.DataFrame) -> Dict[str, Any]:
"""Analyze content structure from crawl data."""
structure_analysis = {
'avg_title_length': 0,
'avg_meta_desc_length': 0,
'h1_usage': 0,
'internal_links_avg': 0,
'external_links_avg': 0
}
# Analyze available columns
if 'title' in crawl_df.columns:
structure_analysis['avg_title_length'] = crawl_df['title'].str.len().mean()
if 'meta_desc' in crawl_df.columns:
structure_analysis['avg_meta_desc_length'] = crawl_df['meta_desc'].str.len().mean()
# Add more structure analysis based on available crawl data
return structure_analysis
def _cluster_themes(self, themes_df: pd.DataFrame) -> Dict[str, List[str]]:
"""Cluster themes into topic groups."""
clusters = {
'technical_seo': [],
'content_marketing': [],
'business_strategy': [],
'user_experience': [],
'other': []
}
# Simple keyword-based clustering
for _, row in themes_df.iterrows():
word = row.get('word', '') if 'word' in row else str(row.get(0, ''))
word_lower = word.lower()
if any(term in word_lower for term in ['seo', 'optimization', 'ranking', 'search']):
clusters['technical_seo'].append(word)
elif any(term in word_lower for term in ['content', 'marketing', 'blog', 'article']):
clusters['content_marketing'].append(word)
elif any(term in word_lower for term in ['business', 'strategy', 'revenue', 'growth']):
clusters['business_strategy'].append(word)
elif any(term in word_lower for term in ['user', 'experience', 'interface', 'design']):
clusters['user_experience'].append(word)
else:
clusters['other'].append(word)
return clusters
async def get_analysis_summary(self, analysis_id: str) -> Dict[str, Any]:
"""
Get analysis summary by ID.
Args:
analysis_id: Analysis identifier
Returns:
Analysis summary
"""
try:
# TODO: Implement database retrieval
return {
'analysis_id': analysis_id,
'status': 'completed',
'summary': 'Analysis completed successfully'
}
except Exception as e:
logger.error(f"Error getting analysis summary: {str(e)}")
return {}
async def health_check(self) -> Dict[str, Any]:
"""
Health check for the content gap analyzer service.
Returns:
Health status
"""
try:
# Test basic functionality
test_keywords = ['test keyword']
test_competitors = ['https://example.com']
# Test SERP analysis
serp_test = await self._analyze_serp_landscape(test_keywords, test_competitors)
# Test keyword expansion
keyword_test = await self._expand_keyword_research(test_keywords, 'test')
# Test competitor analysis
competitor_test = await self._analyze_competitor_content_deep(test_competitors)
return {
'status': 'healthy',
'service': 'ContentGapAnalyzer',
'tests_passed': 3,
'total_tests': 3,
'timestamp': datetime.utcnow().isoformat()
}
except Exception as e:
logger.error(f"Health check failed: {str(e)}")
return {
'status': 'unhealthy',
'service': 'ContentGapAnalyzer',
'error': str(e),
'timestamp': datetime.utcnow().isoformat()
}

1514
backend/services/content_gap_analyzer/keyword_researcher.py Normal file
View File

File diff suppressed because it is too large Load Diff

558
backend/services/content_gap_analyzer/website_analyzer.py Normal file
View File

@@ -0,0 +1,558 @@
"""
Website Analyzer Service
Converted from website_analyzer.py for FastAPI integration.
"""
from typing import Dict, Any, List, Optional
from sqlalchemy.orm import Session
from loguru import logger
from datetime import datetime
import asyncio
import json
from collections import Counter, defaultdict
# Import existing modules (will be updated to use FastAPI services)
from services.database import get_db_session
from .ai_engine_service import AIEngineService
class WebsiteAnalyzer:
"""Analyzes website content structure and performance."""
def __init__(self):
"""Initialize the website analyzer."""
self.ai_engine = AIEngineService()
logger.info("WebsiteAnalyzer initialized")
async def analyze_website(self, url: str, industry: str = "general") -> Dict[str, Any]:
"""
Analyze website content and structure.
Args:
url: Website URL to analyze
industry: Industry category
Returns:
Website analysis results
"""
try:
logger.info(f"Starting website analysis for {url}")
results = {
'website_url': url,
'industry': industry,
'content_analysis': {},
'structure_analysis': {},
'performance_analysis': {},
'seo_analysis': {},
'ai_insights': {},
'analysis_timestamp': datetime.utcnow().isoformat()
}
# Analyze content structure
content_analysis = await self._analyze_content_structure(url)
results['content_analysis'] = content_analysis
# Analyze website structure
structure_analysis = await self._analyze_website_structure(url)
results['structure_analysis'] = structure_analysis
# Analyze performance metrics
performance_analysis = await self._analyze_performance_metrics(url)
results['performance_analysis'] = performance_analysis
# Analyze SEO aspects
seo_analysis = await self._analyze_seo_aspects(url)
results['seo_analysis'] = seo_analysis
# Generate AI insights
ai_insights = await self._generate_ai_insights(results)
results['ai_insights'] = ai_insights
logger.info(f"Website analysis completed for {url}")
return results
except Exception as e:
logger.error(f"Error in website analysis: {str(e)}")
return {}
async def _analyze_content_structure(self, url: str) -> Dict[str, Any]:
"""
Analyze content structure of the website.
Args:
url: Website URL
Returns:
Content structure analysis results
"""
try:
logger.info(f"Analyzing content structure for {url}")
# TODO: Integrate with actual content analysis service
# This will crawl and analyze website content
# Simulate content structure analysis
content_analysis = {
'total_pages': 150,
'content_types': {
'blog_posts': 80,
'product_pages': 30,
'landing_pages': 20,
'guides': 20
},
'content_topics': [
'Industry trends',
'Best practices',
'Case studies',
'Tutorials',
'Expert insights',
'Product information',
'Company news',
'Customer testimonials'
],
'content_depth': {
'shallow': 20,
'medium': 60,
'deep': 70
},
'content_quality_score': 8.5,
'content_freshness': {
'recent': 40,
'moderate': 50,
'outdated': 10
},
'content_engagement': {
'avg_time_on_page': 180,
'bounce_rate': 0.35,
'pages_per_session': 2.5,
'social_shares': 45
}
}
logger.info("Content structure analysis completed")
return content_analysis
except Exception as e:
logger.error(f"Error in content structure analysis: {str(e)}")
return {}
async def _analyze_website_structure(self, url: str) -> Dict[str, Any]:
"""
Analyze website structure and navigation.
Args:
url: Website URL
Returns:
Website structure analysis results
"""
try:
logger.info(f"Analyzing website structure for {url}")
# TODO: Integrate with actual structure analysis service
# This will analyze website architecture and navigation
# Simulate website structure analysis
structure_analysis = {
'navigation_structure': {
'main_menu_items': 8,
'footer_links': 15,
'breadcrumb_usage': True,
'sitemap_available': True
},
'url_structure': {
'avg_url_length': 45,
'seo_friendly_urls': True,
'url_depth': 3,
'canonical_urls': True
},
'internal_linking': {
'avg_internal_links_per_page': 8,
'link_anchor_text_optimization': 75,
'broken_links': 2,
'orphaned_pages': 5
},
'mobile_friendliness': {
'responsive_design': True,
'mobile_optimized': True,
'touch_friendly': True,
'mobile_speed': 85
},
'page_speed': {
'desktop_speed': 85,
'mobile_speed': 75,
'first_contentful_paint': 1.2,
'largest_contentful_paint': 2.5
}
}
logger.info("Website structure analysis completed")
return structure_analysis
except Exception as e:
logger.error(f"Error in website structure analysis: {str(e)}")
return {}
async def _analyze_performance_metrics(self, url: str) -> Dict[str, Any]:
"""
Analyze website performance metrics.
Args:
url: Website URL
Returns:
Performance metrics analysis results
"""
try:
logger.info(f"Analyzing performance metrics for {url}")
# TODO: Integrate with actual performance analysis service
# This will analyze website performance metrics
# Simulate performance metrics analysis
performance_analysis = {
'traffic_metrics': {
'monthly_visitors': '50K+',
'page_views': '150K+',
'unique_visitors': '35K+',
'traffic_growth': '15%'
},
'engagement_metrics': {
'avg_session_duration': '3:45',
'bounce_rate': '35%',
'pages_per_session': 2.5,
'return_visitor_rate': '25%'
},
'conversion_metrics': {
'conversion_rate': '3.5%',
'lead_generation': '500+ monthly',
'sales_conversion': '2.1%',
'email_signups': '200+ monthly'
},
'social_metrics': {
'social_shares': 45,
'social_comments': 12,
'social_engagement_rate': '8.5%',
'social_reach': '10K+'
},
'technical_metrics': {
'page_load_time': 2.1,
'server_response_time': 0.8,
'time_to_interactive': 3.2,
'cumulative_layout_shift': 0.1
}
}
logger.info("Performance metrics analysis completed")
return performance_analysis
except Exception as e:
logger.error(f"Error in performance metrics analysis: {str(e)}")
return {}
async def _analyze_seo_aspects(self, url: str) -> Dict[str, Any]:
"""
Analyze SEO aspects of the website.
Args:
url: Website URL
Returns:
SEO analysis results
"""
try:
logger.info(f"Analyzing SEO aspects for {url}")
# TODO: Integrate with actual SEO analysis service
# This will analyze SEO aspects of the website
# Simulate SEO analysis
seo_analysis = {
'technical_seo': {
'title_tag_optimization': 85,
'meta_description_optimization': 80,
'h1_usage': 95,
'image_alt_text': 70,
'schema_markup': True,
'ssl_certificate': True
},
'on_page_seo': {
'keyword_density': 2.5,
'internal_linking': 8,
'external_linking': 3,
'content_length': 1200,
'readability_score': 75
},
'off_page_seo': {
'domain_authority': 65,
'backlinks': 2500,
'referring_domains': 150,
'social_signals': 45
},
'keyword_rankings': {
'ranking_keywords': 85,
'top_10_rankings': 25,
'top_3_rankings': 8,
'featured_snippets': 3
},
'mobile_seo': {
'mobile_friendly': True,
'mobile_speed': 75,
'mobile_usability': 90,
'amp_pages': 0
},
'local_seo': {
'google_my_business': True,
'local_citations': 45,
'local_keywords': 12,
'local_rankings': 8
}
}
logger.info("SEO analysis completed")
return seo_analysis
except Exception as e:
logger.error(f"Error in SEO analysis: {str(e)}")
return {}
async def _generate_ai_insights(self, analysis_results: Dict[str, Any]) -> Dict[str, Any]:
"""
Generate AI-powered insights for website analysis.
Args:
analysis_results: Complete website analysis results
Returns:
AI-generated insights
"""
try:
logger.info("🤖 Generating AI-powered website insights")
# Prepare analysis summary for AI
analysis_summary = {
'url': analysis_results.get('website_url', ''),
'industry': analysis_results.get('industry', ''),
'content_count': analysis_results.get('content_analysis', {}).get('total_pages', 0),
'content_quality': analysis_results.get('content_analysis', {}).get('content_quality_score', 0),
'performance_score': analysis_results.get('performance_analysis', {}).get('traffic_metrics', {}).get('monthly_visitors', ''),
'seo_score': analysis_results.get('seo_analysis', {}).get('technical_seo', {}).get('title_tag_optimization', 0)
}
# Generate comprehensive AI insights using AI engine
ai_insights = await self.ai_engine.analyze_website_performance(analysis_summary)
if ai_insights:
logger.info("✅ Generated comprehensive AI website insights")
return ai_insights
else:
logger.warning("⚠️ Could not generate AI website insights")
return {}
except Exception as e:
logger.error(f"Error generating AI website insights: {str(e)}")
return {}
async def analyze_content_quality(self, url: str) -> Dict[str, Any]:
"""
Analyze content quality of the website.
Args:
url: Website URL
Returns:
Content quality analysis results
"""
try:
logger.info(f"Analyzing content quality for {url}")
# TODO: Integrate with actual content quality analysis service
# This will analyze content quality metrics
# Simulate content quality analysis
quality_analysis = {
'overall_quality_score': 8.5,
'quality_dimensions': {
'readability': 8.0,
'comprehensiveness': 9.0,
'accuracy': 8.5,
'engagement': 7.5,
'seo_optimization': 8.0
},
'content_strengths': [
'Comprehensive topic coverage',
'Expert-level insights',
'Clear structure and organization',
'Accurate information',
'Good readability'
],
'content_weaknesses': [
'Limited visual content',
'Missing interactive elements',
'Outdated information in some areas',
'Inconsistent content depth'
],
'improvement_areas': [
{
'area': 'Visual Content',
'current_score': 6.0,
'target_score': 9.0,
'improvement_suggestions': [
'Add more images and infographics',
'Include video content',
'Create visual guides',
'Add interactive elements'
]
},
{
'area': 'Content Freshness',
'current_score': 7.0,
'target_score': 9.0,
'improvement_suggestions': [
'Update outdated content',
'Add recent industry insights',
'Include current trends',
'Regular content audits'
]
}
]
}
logger.info("Content quality analysis completed")
return quality_analysis
except Exception as e:
logger.error(f"Error in content quality analysis: {str(e)}")
return {}
async def analyze_user_experience(self, url: str) -> Dict[str, Any]:
"""
Analyze user experience aspects of the website.
Args:
url: Website URL
Returns:
User experience analysis results
"""
try:
logger.info(f"Analyzing user experience for {url}")
# TODO: Integrate with actual UX analysis service
# This will analyze user experience metrics
# Simulate UX analysis
ux_analysis = {
'navigation_experience': {
'menu_clarity': 8.5,
'search_functionality': 7.0,
'breadcrumb_navigation': 9.0,
'mobile_navigation': 8.0
},
'content_accessibility': {
'font_readability': 8.5,
'color_contrast': 9.0,
'alt_text_usage': 7.5,
'keyboard_navigation': 8.0
},
'page_speed_experience': {
'loading_perception': 7.5,
'interactive_elements': 8.0,
'smooth_scrolling': 8.5,
'mobile_performance': 7.0
},
'content_engagement': {
'content_clarity': 8.5,
'call_to_action_visibility': 7.5,
'content_scannability': 8.0,
'information_architecture': 8.5
},
'overall_ux_score': 8.2,
'improvement_suggestions': [
'Improve search functionality',
'Add more visual content',
'Optimize mobile experience',
'Enhance call-to-action visibility'
]
}
logger.info("User experience analysis completed")
return ux_analysis
except Exception as e:
logger.error(f"Error in user experience analysis: {str(e)}")
return {}
async def get_website_summary(self, analysis_id: str) -> Dict[str, Any]:
"""
Get a summary of website analysis.
Args:
analysis_id: Analysis identifier
Returns:
Website analysis summary
"""
try:
logger.info(f"Getting website analysis summary for {analysis_id}")
# TODO: Retrieve analysis from database
# This will be implemented when database integration is complete
summary = {
'analysis_id': analysis_id,
'pages_analyzed': 25,
'content_score': 8.5,
'seo_score': 7.8,
'user_experience_score': 8.2,
'improvement_areas': [
'Content depth and comprehensiveness',
'SEO optimization',
'Mobile responsiveness'
],
'timestamp': datetime.utcnow().isoformat()
}
return summary
except Exception as e:
logger.error(f"Error getting website summary: {str(e)}")
return {}
async def health_check(self) -> Dict[str, Any]:
"""
Health check for the website analyzer service.
Returns:
Health status information
"""
try:
logger.info("Performing health check for WebsiteAnalyzer")
health_status = {
'service': 'WebsiteAnalyzer',
'status': 'healthy',
'dependencies': {
'ai_engine': 'operational'
},
'capabilities': {
'content_analysis': 'operational',
'structure_analysis': 'operational',
'performance_analysis': 'operational',
'seo_analysis': 'operational'
},
'timestamp': datetime.utcnow().isoformat()
}
logger.info("WebsiteAnalyzer health check passed")
return health_status
except Exception as e:
logger.error(f"WebsiteAnalyzer health check failed: {str(e)}")
return {
'service': 'WebsiteAnalyzer',
'status': 'unhealthy',
'error': str(e),
'timestamp': datetime.utcnow().isoformat()
}