kunthawat/ALwrity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Content Calendar, Content Gap Analysis, and Content Optimization

Browse Source
This commit is contained in:
ajaysi
2025-05-27 09:15:08 +05:30
parent 4049d19787
commit 889021c078
100 changed files with 18504 additions and 1251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
lib/utils/alwrity_utils.py
View File

@@ -8,7 +8,7 @@ from loguru import logger
from lib.ai_writers.ai_news_article_writer import ai_news_generation
from lib.ai_writers.ai_financial_writer import write_basic_ta_report
from lib.ai_writers.ai_finance_report_generator.ai_financial_dashboard import get_dashboard
from lib.ai_writers.ai_facebook_writer.facebook_ai_writer import facebook_main_menu
from lib.ai_writers.linkedin_writer.linkedin_ai_writer import linkedin_main_menu
from lib.ai_writers.twitter_writers.twitter_dashboard import run_dashboard
@@ -198,7 +198,9 @@ def ai_finance_ta_writer():
if ticker_symbol:
with st.spinner("Generating TA Report..."):
try:
ta_report = write_basic_ta_report(ticker_symbol)
# Get dashboard instance and generate technical analysis
dashboard = get_dashboard()
ta_report = dashboard.generate_technical_analysis(ticker_symbol)
st.success(f"Successfully generated TA report for: {ticker_symbol}")
st.markdown(ta_report)
except Exception as err:

46
lib/utils/api_key_manager/components/website_setup.py
View File

@@ -3,7 +3,7 @@
import streamlit as st
from loguru import logger
from ...website_analyzer import analyze_website
from ...website_analyzer.seo_analyzer import analyze_seo
from ...website_analyzer.analyzer import WebsiteAnalyzer
import asyncio
import sys
from typing import Dict, Any
@@ -127,37 +127,19 @@ def render_website_setup(api_key_manager: APIKeyManager) -> Dict[str, Any]:
# Call the analyze_website function
results = analyze_website(url)
# If full analysis is selected, add SEO analysis
if analyze_type == "Full Analysis with SEO":
seo_results = analyze_seo(url)
if seo_results.success:
results['data']['seo_analysis'] = {
'overall_score': seo_results.overall_score,
'meta_tags': {
'title': seo_results.meta_tags.title,
'description': seo_results.meta_tags.description,
'keywords': seo_results.meta_tags.keywords,
'has_robots': seo_results.meta_tags.has_robots,
'has_sitemap': seo_results.meta_tags.has_sitemap
},
'content': {
'word_count': seo_results.content.word_count,
'readability_score': seo_results.content.readability_score,
'content_quality_score': seo_results.content.content_quality_score,
'headings_structure': seo_results.content.headings_structure,
'keyword_density': seo_results.content.keyword_density
},
'recommendations': [
{
'priority': rec.priority,
'category': rec.category,
'issue': rec.issue,
'recommendation': rec.recommendation,
'impact': rec.impact
}
for rec in seo_results.recommendations
]
}
# Replace the old SEO analysis code with the new analyzer
analyzer = WebsiteAnalyzer()
seo_results = analyzer.analyze_website(url)
if seo_results.get('success', False):
results['data']['seo_analysis'] = seo_results['data']['analysis']['seo_info']
else:
results['data']['seo_analysis'] = {
'error': seo_results.get('error', 'Unknown error in SEO analysis'),
'overall_score': 0,
'meta_tags': {},
'content': {},
'recommendations': []
}
logger.debug(f"[render_website_setup] Analysis results received: {results.get('success', False)}")

83
lib/utils/save_to_file.py Normal file
View File

@@ -0,0 +1,83 @@
"""
Utility module for saving generated content to files.
Handles saving various types of content to the workspace directory.
"""
import os
import json
from pathlib import Path
from datetime import datetime
from typing import Union, Dict, List, Any
# Define the workspace directory
WORKSPACE_DIR = Path(__file__).parent.parent.parent / "workspace" / "alwrity_content"
def ensure_directory_exists(directory: Union[str, Path]) -> None:
"""Ensure the specified directory exists."""
os.makedirs(directory, exist_ok=True)
def save_to_file(
content: Union[str, Dict, List, Any],
filename: str,
content_type: str = "text",
subdirectory: str = None
) -> str:
"""
Save content to a file in the workspace directory.
Args:
content: The content to save (string, dict, list, or any serializable object)
filename: Name of the file to save
content_type: Type of content ('text', 'json', 'audio', 'image')
subdirectory: Optional subdirectory within the workspace
Returns:
str: Path to the saved file
"""
# Create timestamp for unique filenames
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
base_filename = f"{timestamp}_{filename}"
# Determine the target directory
target_dir = WORKSPACE_DIR
if subdirectory:
target_dir = target_dir / subdirectory
# Ensure directory exists
ensure_directory_exists(target_dir)
# Determine file extension and format content
if content_type == "json":
file_path = target_dir / f"{base_filename}.json"
with open(file_path, "w", encoding="utf-8") as f:
json.dump(content, f, indent=2, ensure_ascii=False)
elif content_type == "audio":
file_path = target_dir / f"{base_filename}.mp3"
with open(file_path, "wb") as f:
f.write(content)
elif content_type == "image":
file_path = target_dir / f"{base_filename}.png"
with open(file_path, "wb") as f:
f.write(content)
else: # text
file_path = target_dir / f"{base_filename}.txt"
with open(file_path, "w", encoding="utf-8") as f:
f.write(str(content))
return str(file_path)
def save_audio(audio_bytes: bytes, filename: str, subdirectory: str = "audio") -> str:
"""Save audio content to a file."""
return save_to_file(audio_bytes, filename, "audio", subdirectory)
def save_image(image_bytes: bytes, filename: str, subdirectory: str = "images") -> str:
"""Save image content to a file."""
return save_to_file(image_bytes, filename, "image", subdirectory)
def save_json(data: Union[Dict, List], filename: str, subdirectory: str = "json") -> str:
"""Save JSON content to a file."""
return save_to_file(data, filename, "json", subdirectory)
def save_text(text: str, filename: str, subdirectory: str = "text") -> str:
"""Save text content to a file."""
return save_to_file(text, filename, "text", subdirectory)

484
lib/utils/seo_tools.py
View File

@@ -9,7 +9,347 @@ from lib.ai_seo_tools.google_pagespeed_insights import google_pagespeed_insights
from lib.ai_seo_tools.on_page_seo_analyzer import analyze_onpage_seo
from lib.ai_seo_tools.weburl_seo_checker import url_seo_checker
from lib.ai_marketing_tools.ai_backlinker.backlinking_ui_streamlit import backlinking_ui
from lib.ai_seo_tools.content_gap_analysis.ui import ContentGapAnalysisUI
from lib.ai_seo_tools.content_calendar.ui.dashboard import ContentCalendarDashboard
def render_content_gap_analysis():
"""Render the content gap analysis workflow interface."""
from lib.ai_seo_tools.content_gap_analysis.ui import ContentGapAnalysisUI
# Initialize and run the Content Gap Analysis UI
ui = ContentGapAnalysisUI()
ui.run()
def render_content_calendar():
"""Render the content calendar dashboard."""
import logging
import sys
from datetime import datetime
# Configure logging
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout),
logging.FileHandler('content_calendar.log', mode='a')
]
)
logger = logging.getLogger('content_calendar')
try:
logger.info("Initializing Content Calendar Dashboard")
dashboard = ContentCalendarDashboard()
logger.info("Rendering Content Calendar Dashboard")
dashboard.render()
logger.info("Content Calendar Dashboard rendered successfully")
except Exception as e:
logger.error(f"Error rendering content calendar: {str(e)}", exc_info=True)
st.error(f"An error occurred while loading the content calendar: {str(e)}")
def render_seo_tools_dashboard():
"""Render a modern dashboard for SEO tools with improved UI and navigation."""
selected_section = st.session_state.get('seo_dashboard_section', 'combinations')
# Define card gradients at the top so it's available in all sections
card_gradients = {
"Content Optimization Suite": "linear-gradient(135deg, #43e97b 0%, #38f9d7 100%)",
"Technical SEO Audit": "linear-gradient(135deg, #667eea 0%, #764ba2 100%)",
"Image Optimization Suite": "linear-gradient(135deg, #f7971e 0%, #ffd200 100%)",
"Social Media Optimization": "linear-gradient(135deg, #f953c6 0%, #b91d73 100%)",
"Content Gap Analysis": "linear-gradient(135deg, #667eea 0%, #764ba2 100%)",
"Content Calendar": "linear-gradient(135deg, #4CAF50 0%, #2196F3 100%)",
"Structured Data Generator": "linear-gradient(135deg, #43e97b 0%, #38f9d7 100%)",
"Blog Title Generator": "linear-gradient(135deg, #2193b0 0%, #6dd5ed 100%)",
"Meta Description Generator": "linear-gradient(135deg, #f7971e 0%, #ffd200 100%)",
"Image Alt Text Generator": "linear-gradient(135deg, #f953c6 0%, #b91d73 100%)",
"OpenGraph Tags Generator": "linear-gradient(135deg, #f857a6 0%, #ff5858 100%)",
"Image Optimizer": "linear-gradient(135deg, #43cea2 0%, #185a9d 100%)",
"PageSpeed Insights": "linear-gradient(135deg, #ff9966 0%, #ff5e62 100%)",
"On-Page SEO Analyzer": "linear-gradient(135deg, #56ab2f 0%, #a8e063 100%)",
"URL SEO Checker": "linear-gradient(135deg, #3a7bd5 0%, #00d2ff 100%)",
"AI Backlinking Tool": "linear-gradient(135deg, #e96443 0%, #904e95 100%)"
}
# Navigation bar only (no dashboard title/description)
nav_cols = st.columns([1,1,1,1])
nav_labels = ["Tool Combos", "Advanced", "Individual", "About"]
nav_keys = ["combinations", "advanced", "individual", "about"]
for i, label in enumerate(nav_labels):
if nav_cols[i].button(label, key=f"nav_{label}"):
st.session_state['seo_dashboard_section'] = nav_keys[i]
selected_section = nav_keys[i]
st.markdown("<hr style='margin:1.5rem 0;'>", unsafe_allow_html=True)
# Define tool combinations for cross-tool analysis
tool_combinations = {
"Content Optimization Suite": {
"icon": "📊",
"description": "Comprehensive content optimization combining title generation, meta descriptions, and structured data.",
"tools": ["Blog Title Generator", "Meta Description Generator", "Structured Data Generator"],
"path": "content_optimization",
"color": "#4CAF50"
},
"Technical SEO Audit": {
"icon": "🔧",
"description": "Complete technical SEO analysis including page speed, on-page SEO, and URL structure.",
"tools": ["PageSpeed Insights", "On-Page SEO Analyzer", "URL SEO Checker"],
"path": "technical_audit",
"color": "#2196F3"
},
"Image Optimization Suite": {
"icon": "🖼️",
"description": "End-to-end image optimization with alt text generation and performance optimization.",
"tools": ["Image Alt Text Generator", "Image Optimizer"],
"path": "image_optimization",
"color": "#FF9800"
},
"Social Media Optimization": {
"icon": "📱",
"description": "Enhance social media presence with OpenGraph tags and backlink analysis.",
"tools": ["OpenGraph Tags Generator", "AI Backlinking Tool"],
"path": "social_optimization",
"color": "#9C27B0"
}
}
# Define individual SEO tools
seo_tools = {
"Structured Data Generator": {
"icon": "📋",
"description": "Generate structured data (Rich Snippets) to enhance your search results with additional information.",
"color": "#4CAF50",
"path": "structured_data",
"status": "active"
},
"Blog Title Generator": {
"icon": "✏️",
"description": "Create SEO-optimized blog titles that attract clicks and improve search rankings.",
"color": "#2196F3",
"path": "blog_title",
"status": "active"
},
"Meta Description Generator": {
"icon": "📝",
"description": "Generate compelling meta descriptions that improve click-through rates from search results.",
"color": "#FF9800",
"path": "meta_description",
"status": "active"
},
"Image Alt Text Generator": {
"icon": "🖼️",
"description": "Create descriptive alt text for images to improve accessibility and image SEO.",
"color": "#9C27B0",
"path": "alt_text",
"status": "active"
},
"OpenGraph Tags Generator": {
"icon": "📱",
"description": "Generate OpenGraph tags for better social media sharing and visibility.",
"color": "#F44336",
"path": "opengraph",
"status": "active"
},
"Image Optimizer": {
"icon": "📉",
"description": "Optimize and resize images for better website performance and SEO.",
"color": "#607D8B",
"path": "image_optimizer",
"status": "active"
},
"PageSpeed Insights": {
"icon": "⚡",
"description": "Analyze your website's performance using Google PageSpeed Insights.",
"color": "#795548",
"path": "pagespeed",
"status": "active"
},
"On-Page SEO Analyzer": {
"icon": "🔍",
"description": "Analyze and optimize your webpage's SEO elements and content.",
"color": "#009688",
"path": "onpage_seo",
"status": "active"
},
"URL SEO Checker": {
"icon": "🌐",
"description": "Check the SEO health of specific URLs and get improvement suggestions.",
"color": "#3F51B5",
"path": "url_checker",
"status": "active"
},
"AI Backlinking Tool": {
"icon": "🔗",
"description": "Discover and analyze backlink opportunities using AI-powered insights.",
"color": "#E91E63",
"path": "backlinking",
"status": "active"
}
}
# --- Tool Combinations Section ---
if selected_section == 'combinations':
combo_cols = st.columns(2)
for idx, (combo_name, details) in enumerate(tool_combinations.items()):
gradient = card_gradients.get(combo_name, "linear-gradient(135deg, #f5f7fa 0%, #c3cfe2 100%)")
with combo_cols[idx % 2]:
st.markdown(f"""
<div class="seo-card" style="background: {gradient}; position: relative; overflow: hidden;">
<div class="seo-card-overlay"></div>
<div class="seo-icon">{details['icon']}</div>
<div class="seo-title">{combo_name}</div>
<div class="seo-description">{details['description']}</div>
<div>
{''.join([f'<span class="tool-badge">{tool}</span>' for tool in details['tools']])}
</div>
</div>
""", unsafe_allow_html=True)
if st.button(f"Launch {combo_name}", key=f"combo_{combo_name}", use_container_width=True):
st.query_params["tool"] = details["path"]
st.rerun()
# --- Advanced Features Section ---
elif selected_section == 'advanced':
adv_cols = st.columns(2)
adv_features = [
{
"name": "Content Gap Analysis",
"icon": "🎯",
"description": "Identify content opportunities and optimize your content strategy with AI-powered insights.",
"badges": ["Website Analysis", "Competitor Research", "Keyword Opportunities", "AI Recommendations"],
"gradient": card_gradients["Content Gap Analysis"],
"button": "Start Content Gap Analysis",
"key": "content_gap_analysis",
"path": "content_gap_analysis"
},
{
"name": "Content Calendar",
"icon": "📅",
"description": "Plan, schedule, and manage your content strategy with our AI-powered content calendar.",
"badges": ["Content Planning", "Scheduling", "Performance Tracking", "AI Insights"],
"gradient": card_gradients["Content Calendar"],
"button": "Open Content Calendar",
"key": "content_calendar",
"path": "content_calendar"
}
]
for idx, feature in enumerate(adv_features):
with adv_cols[idx % 2]:
st.markdown(f"""
<div class="seo-card" style="background: {feature['gradient']}; position: relative; overflow: hidden;">
<div class="seo-card-overlay"></div>
<div class="seo-icon">{feature['icon']}</div>
<div class="seo-title">{feature['name']}</div>
<div class="seo-description">{feature['description']}</div>
<div>
{''.join([f'<span class="tool-badge">{badge}</span>' for badge in feature['badges']])}
</div>
</div>
""", unsafe_allow_html=True)
if st.button(feature['button'], key=feature['key'], use_container_width=True):
st.query_params["tool"] = feature["path"]
st.rerun()
# --- Individual Tools Section ---
elif selected_section == 'individual':
cols = st.columns(3)
for idx, (tool_name, details) in enumerate(seo_tools.items()):
gradient = card_gradients.get(tool_name, "linear-gradient(135deg, #f5f7fa 0%, #c3cfe2 100%)")
with cols[idx % 3]:
st.markdown(f"""
<div class="seo-card" style="background: {gradient}; position: relative; overflow: hidden;">
<div class="seo-card-overlay"></div>
<div class="seo-icon">{details['icon']}</div>
<div class="seo-title">{tool_name}</div>
<div class="seo-description">{details['description']}</div>
</div>
""", unsafe_allow_html=True)
if st.button(f"Use {tool_name}", key=f"btn_{tool_name}", use_container_width=True):
st.query_params["tool"] = details["path"]
st.rerun()
# --- About Section ---
elif selected_section == 'about':
st.markdown("""
<div style='text-align: center; margin: 2rem 0;'>
<h2>About This Dashboard</h2>
<p style='color: #666;'>This dashboard brings together powerful AI-driven SEO tools and workflows to help you optimize your website and content strategy. Use the navigation above to explore combinations, advanced features, or individual tools.</p>
</div>
""", unsafe_allow_html=True)
st.markdown("""
<style>
.seo-card {
border-radius: 14px;
padding: 24px;
margin-bottom: 24px;
box-shadow: 0 4px 16px rgba(44, 62, 80, 0.10), 0 1.5px 4px rgba(44,62,80,0.06);
transition: transform 0.2s cubic-bezier(.4,2,.6,1), box-shadow 0.2s;
height: 100%;
border: 1.5px solid #e3e8ee;
position: relative;
overflow: hidden;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
}
.seo-card-overlay {
position: absolute;
top: 0; left: 0; right: 0; bottom: 0;
background: rgba(255,255,255,0.72);
z-index: 1;
pointer-events: none;
border-radius: 14px;
box-shadow: 0 2px 8px rgba(44,62,80,0.08);
}
.seo-card:hover {
transform: translateY(-6px) scale(1.025);
box-shadow: 0 8px 32px rgba(44, 62, 80, 0.18), 0 2px 8px rgba(44,62,80,0.10);
border-color: #4CAF50;
}
.seo-icon {
font-size: 2.7rem;
margin-bottom: 18px;
z-index: 2;
position: relative;
text-shadow: 0 2px 8px rgba(44,62,80,0.10);
}
.seo-title {
font-size: 1.25rem;
font-weight: 800;
margin-bottom: 12px;
color: #222b45;
z-index: 2;
position: relative;
text-shadow: 0 2px 8px rgba(44,62,80,0.10);
letter-spacing: 0.01em;
}
.seo-description {
color: #34495e;
font-size: 1.08rem;
margin-bottom: 15px;
z-index: 2;
position: relative;
text-align: center;
line-height: 1.5;
text-shadow: 0 1px 4px rgba(44,62,80,0.08);
}
.tool-badge {
display: inline-block;
padding: 4px 12px;
border-radius: 12px;
font-size: 0.9rem;
margin-right: 8px;
margin-bottom: 8px;
background: rgba(255, 255, 255, 0.95);
color: #2196F3;
font-weight: 600;
border: 1px solid #e3e8ee;
}
</style>
""", unsafe_allow_html=True)
def ai_seo_tools():
"""
@@ -17,71 +357,83 @@ def ai_seo_tools():
such as generating structured data, optimizing images, checking page speed,
and analyzing on-page SEO.
"""
st.markdown(
"""
Welcome to your one-stop solution for AI-driven SEO optimization. Select a tool from the options below
to improve your website’s SEO with cutting-edge AI technology.
"""
)
# List of SEO tools with unique emojis for each option
options = [
"📝 Generate Structured Data - Rich Snippet",
"✏️ Generate SEO Optimized Blog Titles",
"📝 Generate Meta Description for SEO",
"🖼️ Generate Image Alt Text",
"📄 Generate OpenGraph Tags",
"📉 Optimize/Resize Image",
"⚡ Run Google PageSpeed Insights",
"🔍 Analyze On-Page SEO",
"🌐 URL SEO Checker",
"🔗 AI Backlinking Tool"
]
# Check if a specific tool is selected
selected_tool = st.query_params.get("tool")
# User selection of SEO tools using radio buttons
choice = st.radio(
"**👇 Select an AI SEO Tool:**",
options,
index=0,
format_func=lambda x: x
)
if selected_tool:
# Map tool paths to their respective functions
tool_functions = {
# Individual tools
"structured_data": ai_structured_data,
"blog_title": ai_title_generator,
"meta_description": metadesc_generator_main,
"alt_text": alt_text_gen,
"opengraph": og_tag_generator,
"image_optimizer": main_img_optimizer,
"pagespeed": google_pagespeed_insights,
"onpage_seo": analyze_onpage_seo,
"url_checker": url_seo_checker,
"backlinking": backlinking_ui,
# Tool combinations
"content_optimization": lambda: run_tool_combination([
ai_title_generator,
metadesc_generator_main,
ai_structured_data
], "Content Optimization Suite"),
"technical_audit": lambda: run_tool_combination([
google_pagespeed_insights,
analyze_onpage_seo,
url_seo_checker
], "Technical SEO Audit"),
"image_optimization": lambda: run_tool_combination([
alt_text_gen,
main_img_optimizer
], "Image Optimization Suite"),
"social_optimization": lambda: run_tool_combination([
og_tag_generator,
backlinking_ui
], "Social Media Optimization"),
# Add Content Gap Analysis and Content Calendar
"content_gap_analysis": render_content_gap_analysis,
"content_calendar": render_content_calendar
}
if selected_tool in tool_functions:
# Clear any existing content
st.empty()
# Execute the selected tool's function
tool_functions[selected_tool]()
else:
st.error(f"Invalid tool selected: {selected_tool}")
render_seo_tools_dashboard()
else:
# Show the dashboard if no tool is selected
render_seo_tools_dashboard()
def run_tool_combination(tools, combination_name):
"""Run a combination of tools and provide cross-tool analysis."""
st.markdown(f"# {combination_name}")
st.markdown("Running comprehensive analysis...")
# Call the respective functions based on the user selection
if choice == "📝 Generate Structured Data - Rich Snippet":
# Generate Structured Data for Rich Snippets
ai_structured_data()
elif choice == "📝 Generate Meta Description for SEO":
# Generate SEO-optimized meta descriptions
metadesc_generator_main()
elif choice == "✏️ Generate SEO Optimized Blog Titles":
# Generate SEO-friendly blog titles
ai_title_generator()
elif choice == "🖼️ Generate Image Alt Text":
# Generate alternative text for images
alt_text_gen()
elif choice == "📄 Generate OpenGraph Tags":
# Generate OpenGraph tags for social media sharing
og_tag_generator()
elif choice == "📉 Optimize/Resize Image":
# Optimize images by resizing or compressing them
main_img_optimizer()
elif choice == "⚡ Run Google PageSpeed Insights":
# Run Google PageSpeed Insights for performance analysis
google_pagespeed_insights()
elif choice == "🔍 Analyze On-Page SEO":
# Analyze on-page SEO elements
analyze_onpage_seo()
elif choice == "🌐 URL SEO Checker":
# Check SEO health of a specific URL
url_seo_checker()
elif choice == "🔗 AI Backlinking Tool":
# Run AI Backlinking tool for link-building opportunities
backlinking_ui()
# Create tabs for each tool in the combination
tabs = st.tabs([f"Step {i+1}" for i in range(len(tools))])
# Run each tool in its own tab
for i, (tab, tool) in enumerate(zip(tabs, tools)):
with tab:
st.markdown(f"### Step {i+1}")
tool()
# Add cross-tool analysis section
st.markdown("## 📊 Cross-Tool Analysis")
st.markdown("Analyzing results across all tools...")
# Add recommendations based on combined results
st.markdown("## 💡 Recommendations")
st.markdown("Based on the combined analysis, here are the key recommendations:")
# Add a button to export the complete analysis
if st.button("📥 Export Complete Analysis", use_container_width=True):
st.info("Analysis export functionality coming soon!")

234
lib/utils/ui_setup.py
View File

@@ -1,3 +1,8 @@
"""
UI setup module for ALwrity application.
Provides consistent navigation and layout structure.
"""
import os
import streamlit as st
from lib.utils.file_processor import load_image
@@ -15,6 +20,99 @@ from lib.ai_writers.insta_ai_writer import insta_writer
from lib.ai_writers.youtube_writers.youtube_ai_writer import youtube_main_menu
from lib.ai_writers.ai_writer_dashboard import get_ai_writers, list_ai_writers
def render_social_tools_dashboard():
"""Render a modern dashboard for social media tools."""
st.markdown("""
<style>
.social-card {
background: white;
border-radius: 10px;
padding: 20px;
box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
transition: transform 0.3s ease;
height: 100%;
}
.social-card:hover {
transform: translateY(-5px);
}
.social-icon {
font-size: 2.5rem;
margin-bottom: 15px;
}
.social-title {
font-size: 1.2rem;
font-weight: 600;
margin-bottom: 10px;
}
.social-description {
color: #666;
font-size: 0.9rem;
margin-bottom: 15px;
}
.social-button {
width: 100%;
padding: 8px 16px;
border-radius: 5px;
border: none;
font-weight: 500;
cursor: pointer;
transition: all 0.3s ease;
}
</style>
""", unsafe_allow_html=True)
# Define social tools with their details and paths
social_tools = {
"Facebook": {
"icon": "📘",
"description": "Create engaging Facebook posts and manage your content strategy",
"color": "#4267B2",
"path": "facebook"
},
"LinkedIn": {
"icon": "💼",
"description": "Generate professional LinkedIn content and optimize your profile",
"color": "#0077B5",
"path": "linkedin"
},
"Twitter": {
"icon": "🐦",
"description": "Craft viral tweets and manage your Twitter presence",
"color": "#1DA1F2",
"path": "twitter"
},
"Instagram": {
"icon": "📸",
"description": "Create Instagram captions and plan your visual content",
"color": "#E1306C",
"path": "instagram"
},
"YouTube": {
"icon": "🎥",
"description": "Generate video scripts and optimize your YouTube content",
"color": "#FF0000",
"path": "youtube"
}
}
# Create a grid of cards
cols = st.columns(3)
for idx, (platform, details) in enumerate(social_tools.items()):
with cols[idx % 3]:
st.markdown(f"""
<div class="social-card">
<div class="social-icon">{details['icon']}</div>
<div class="social-title">{platform}</div>
<div class="social-description">{details['description']}</div>
</div>
""", unsafe_allow_html=True)
if st.button(f"Open {platform}", key=f"btn_{platform}",
help=f"Launch {platform} tools",
use_container_width=True):
# Set query parameters to redirect to the specific tool
st.query_params["tool"] = details["path"]
st.rerun()
def setup_ui():
"""Set up the UI with custom styling."""
@@ -314,29 +412,18 @@ def setup_alwrity_ui():
"AI Writers": ("📝", get_ai_writers),
"Content Planning": ("📅", content_planning_tools),
"AI SEO Tools": ("🔍", ai_seo_tools),
"AI Social Tools": ("📱", None), # Set to None as we'll handle this separately
"AI Social Tools": ("📱", render_social_tools_dashboard),
"ALwrity Settings": ("⚙️", render_settings_page),
"Agents Teams(TBD)": ("🤝", lambda: st.subheader("Agents Teams - Coming Soon!")),
"Ask Alwrity(TBD)": ("💬", lambda: (
st.subheader("Chat with your Data, Chat with any Data.. COMING SOON !"),
st.markdown("Create a collection by uploading files (PDF, MD, CSV, etc), or crawl a data source (Websites, more sources coming soon."),
st.markdown("One can ask/chat, summarize and do semantic search over the uploaded data")
)),
"ALwrity Settings": ("⚙️", render_settings_page)
))
}
logger.info(f"Defined {len(nav_items)} navigation items")
# Define sub-menu items for AI Social Tools
social_tools_submenu = {
"Facebook": ("📘", lambda: facebook_main_menu()),
"LinkedIn": ("💼", lambda: linkedin_main_menu()),
"Twitter": ("🐦", lambda: run_dashboard()),
"Instagram": ("📸", lambda: insta_writer()),
"YouTube": ("🎥", lambda: youtube_main_menu())
}
logger.info(f"Defined {len(social_tools_submenu)} social tools submenu items")
# Create sidebar navigation
st.sidebar.markdown("### ALwrity Options")
st.sidebar.markdown('<div class="sidebar-nav">', unsafe_allow_html=True)
@@ -345,53 +432,12 @@ def setup_alwrity_ui():
for name, (icon, func) in nav_items.items():
button_class = "nav-button active" if st.session_state.active_tab == name else "nav-button"
if name == "AI Social Tools":
# For AI Social Tools, we'll create a button that toggles the sub-menu
if st.sidebar.button(f"{icon} {name}", key=f"nav_{name}",
help=f"Navigate to {name}", use_container_width=True):
st.session_state.active_tab = name
# Reset sub-tab when main tab changes
st.session_state.active_sub_tab = None
logger.info(f"Selected main tab: {name}")
# If AI Social Tools is active, show the sub-menu
if st.session_state.active_tab == "AI Social Tools":
st.sidebar.markdown('<div class="sub-menu">', unsafe_allow_html=True)
# Create sub-menu buttons
for sub_name, (sub_icon, sub_func) in social_tools_submenu.items():
# Create the button with a custom key that includes the platform name
button_key = f"sub_{sub_name}"
# Determine if this button is active
is_active = st.session_state.active_sub_tab == sub_name
# Create a container with the platform-specific class
platform_class = f"{sub_name.lower()}-button"
if is_active:
platform_class += " active"
# Add the platform-specific class to the button container
st.sidebar.markdown(f'<div class="{platform_class}">', unsafe_allow_html=True)
# Create the button
if st.sidebar.button(f"{sub_icon} {sub_name}", key=button_key,
help=f"Navigate to {sub_name}", use_container_width=True):
st.session_state.active_sub_tab = sub_name
logger.info(f"Selected social tool: {sub_name}")
# Close the div
st.sidebar.markdown('</div>', unsafe_allow_html=True)
st.sidebar.markdown('</div>', unsafe_allow_html=True)
else:
# For other navigation items, create regular buttons
if st.sidebar.button(f"{icon} {name}", key=f"nav_{name}",
help=f"Navigate to {name}", use_container_width=True):
st.session_state.active_tab = name
# Reset sub-tab when main tab changes
st.session_state.active_sub_tab = None
logger.info(f"Selected main tab: {name}")
if st.sidebar.button(f"{icon} {name}", key=f"nav_{name}",
help=f"Navigate to {name}", use_container_width=True):
st.session_state.active_tab = name
# Reset sub-tab when main tab changes
st.session_state.active_sub_tab = None
logger.info(f"Selected main tab: {name}")
st.sidebar.markdown('</div>', unsafe_allow_html=True)
@@ -402,10 +448,36 @@ def setup_alwrity_ui():
st.sidebar.image(icon_path, use_container_width=False)
st.sidebar.markdown('</div>', unsafe_allow_html=True)
# Display content based on active tab
# Display content based on active tab and tool selection
if st.session_state.active_tab == "AI Social Tools":
if not st.session_state.active_sub_tab:
# Only show title and info when no sub-tab is selected
# Check if a specific tool is selected
selected_tool = st.query_params.get("tool")
if selected_tool:
# Add a back button at the top
if st.button("← Back to Social Tools Dashboard", key=f"back_to_dashboard_{selected_tool}"):
# Clear the tool query parameter
st.query_params.clear()
st.rerun()
# Map tool paths to their respective functions
tool_functions = {
"facebook": facebook_main_menu,
"linkedin": linkedin_main_menu,
"twitter": run_dashboard,
"instagram": insta_writer,
"youtube": youtube_main_menu
}
if selected_tool in tool_functions:
# Clear any existing content
st.empty()
# Execute the selected tool's function
tool_functions[selected_tool]()
else:
st.error(f"Invalid tool selected: {selected_tool}")
render_social_tools_dashboard()
else:
# Show the dashboard if no tool is selected
st.markdown("""
<style>
.main .block-container {
@@ -414,40 +486,14 @@ def setup_alwrity_ui():
</style>
""", unsafe_allow_html=True)
st.title(f"{nav_items[st.session_state.active_tab][0]} {st.session_state.active_tab}")
st.info("Please select a social media platform from the sidebar.")
else:
# When a platform is selected, show no title and minimize spacing
st.markdown("""
<style>
.main .block-container {
padding-top: 0 !important;
padding-bottom: 0;
}
/* Remove all margins and padding from content area */
.element-container {
margin: 0 !important;
padding: 0 !important;
}
/* Hide any automatic headers */
.main .block-container > div:first-child {
margin-top: 0 !important;
padding-top: 0 !important;
}
</style>
""", unsafe_allow_html=True)
# Call the function directly without any title
social_tools_submenu[st.session_state.active_sub_tab][1]()
render_social_tools_dashboard()
else:
# Check if we're in the AI Writers section and handle writer selection
# Handle other tabs as before
if st.session_state.active_tab == "AI Writers":
# Get the writer parameter from the URL using st.query_params
writer = st.query_params.get("writer")
logger.info(f"Current writer from query params: {writer}")
if writer:
# Get the list of writers without rendering the dashboard
writers = list_ai_writers()
logger.info(f"Found {len(writers)} writers")
@@ -457,9 +503,7 @@ def setup_alwrity_ui():
if w["path"] == writer:
writer_found = True
logger.info(f"Found matching writer: {w['name']}, executing function")
# Clear any existing content
st.empty()
# Execute the writer function
w["function"]()
break
@@ -467,11 +511,9 @@ def setup_alwrity_ui():
logger.error(f"No writer found with path: {writer}")
st.error(f"No writer found with path: {writer}")
else:
# If no writer selected, show the dashboard
logger.info("No writer selected, showing dashboard")
get_ai_writers()
else:
# For all other tabs, show the title
st.markdown("""
<style>
.main .block-container {

5
lib/utils/website_analyzer/__init__.py
View File

@@ -1,7 +1,6 @@
"""Website analyzer module for AI-powered website analysis."""
from .analyzer import analyze_website
from .seo_analyzer import analyze_seo
from .analyzer import analyze_website, WebsiteAnalyzer
from .models import SEOAnalysisResult
__all__ = ['analyze_seo', 'SEOAnalysisResult', 'analyze_website']
__all__ = ['analyze_website', 'WebsiteAnalyzer', 'SEOAnalysisResult']

486
lib/utils/website_analyzer/analyzer.py
View File

@@ -1,7 +1,7 @@
"""Website scraping and AI analysis module."""
"""Website and SEO analysis module."""
import asyncio
from typing import Dict, List, Optional
from typing import Dict, List, Optional, Tuple
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import streamlit as st
@@ -21,51 +21,29 @@ import whois
import dns.resolver
from requests.exceptions import RequestException
from concurrent.futures import ThreadPoolExecutor
from .models import (
SEOAnalysisResult,
MetaTagAnalysis,
ContentAnalysis,
SEORecommendation
)
# Configure logging
logging.basicConfig(
level=logging.DEBUG,
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(),
logging.FileHandler('logs/website_analyzer.log')
]
)
# Create a logger for the website analyzer
logger = logging.getLogger(__name__)
def analyze_website(url: str) -> Dict:
"""
Analyze a website and return comprehensive results.
Args:
url (str): The URL to analyze
Returns:
Dict: Analysis results including various metrics and checks
"""
logger.info(f"Starting website analysis for URL: {url}")
try:
analyzer = WebsiteAnalyzer()
results = analyzer.analyze_website(url)
# Add success status to results
if "error" in results:
return {
"success": False,
"error": results["error"]
}
# Add success status and wrap results
return {
"success": True,
"data": results
}
except Exception as e:
logger.error(f"Error in analyze_website: {str(e)}", exc_info=True)
return {
"success": False,
"error": str(e)
}
# Create a separate logger for scraping operations
scraping_logger = logging.getLogger('website_analyzer.scraping')
scraping_logger.setLevel(logging.WARNING)
class WebsiteAnalyzer:
def __init__(self):
@@ -89,13 +67,17 @@ class WebsiteAnalyzer:
try:
# Validate URL
if not self._validate_url(url):
logger.error(f"Invalid URL format: {url}")
return {"error": "Invalid URL format"}
error_msg = f"Invalid URL format: {url}"
logger.error(error_msg)
return {
"success": False,
"error": error_msg,
"error_details": {"stage": "url_validation"}
}
# Basic URL parsing
parsed_url = urlparse(url)
domain = parsed_url.netloc
logger.debug(f"Parsed domain: {domain}")
# Initialize results dictionary
results = {
@@ -107,36 +89,105 @@ class WebsiteAnalyzer:
# Perform various analyses
with ThreadPoolExecutor(max_workers=4) as executor:
logger.info("Starting parallel analysis tasks")
# Basic website info
logger.info("Starting basic info analysis")
basic_info = executor.submit(self._get_basic_info, url).result()
if "error" in basic_info:
error_msg = f"Basic info analysis failed: {basic_info['error']}"
logger.error(error_msg)
return {
"success": False,
"error": error_msg,
"error_details": {
"stage": "basic_info",
"details": basic_info.get("error_details", {})
}
}
results["analysis"]["basic_info"] = basic_info
# SSL/TLS info
logger.info("Starting SSL analysis")
ssl_info = executor.submit(self._check_ssl, domain).result()
results["analysis"]["ssl_info"] = ssl_info
# DNS info
logger.info("Starting DNS analysis")
dns_info = executor.submit(self._check_dns, domain).result()
results["analysis"]["dns_info"] = dns_info
# WHOIS info
logger.info("Starting WHOIS analysis")
whois_info = executor.submit(self._get_whois_info, domain).result()
results["analysis"]["whois_info"] = whois_info
# Content analysis
logger.info("Starting content analysis")
content_info = executor.submit(self._analyze_content, url).result()
if "error" in content_info:
error_msg = f"Content analysis failed: {content_info['error']}"
logger.error(error_msg)
return {
"success": False,
"error": error_msg,
"error_details": {
"stage": "content_analysis",
"details": content_info.get("error_details", {})
}
}
results["analysis"]["content_info"] = content_info
# Performance metrics
logger.info("Starting performance analysis")
performance = executor.submit(self._check_performance, url).result()
if "error" in performance:
error_msg = f"Performance analysis failed: {performance['error']}"
logger.error(error_msg)
return {
"success": False,
"error": error_msg,
"error_details": {
"stage": "performance_analysis",
"details": performance.get("error_details", {})
}
}
results["analysis"]["performance"] = performance
# SEO analysis
logger.info("Starting SEO analysis")
seo_analysis = executor.submit(self._analyze_seo, url).result()
if "error" in seo_analysis:
error_msg = f"SEO analysis failed: {seo_analysis['error']}"
logger.error(error_msg)
return {
"success": False,
"error": error_msg,
"error_details": {
"stage": "seo_analysis",
"details": seo_analysis.get("error_details", {})
}
}
results["analysis"]["seo_info"] = seo_analysis
logger.info(f"Analysis completed successfully for {url}")
return results
logger.debug(f"Final results: {json.dumps(results, indent=2)}")
return {
"success": True,
"data": results
}
except Exception as e:
logger.error(f"Error during website analysis: {str(e)}", exc_info=True)
return {"error": str(e)}
error_msg = f"Error during website analysis: {str(e)}"
logger.error(error_msg, exc_info=True)
return {
"success": False,
"error": error_msg,
"error_details": {
"type": type(e).__name__,
"traceback": str(e.__traceback__)
}
}
def _validate_url(self, url: str) -> bool:
"""Validate URL format."""
@@ -149,7 +200,7 @@ class WebsiteAnalyzer:
def _get_basic_info(self, url: str) -> Dict:
"""Get basic website information."""
logger.debug(f"Getting basic info for {url}")
scraping_logger.debug(f"Getting basic info for {url}")
try:
response = self.session.get(url, timeout=10)
response.raise_for_status()
@@ -165,13 +216,31 @@ class WebsiteAnalyzer:
"robots_txt": self._get_robots_txt(url),
"sitemap": self._get_sitemap(url)
}
except requests.exceptions.RequestException as e:
error_msg = f"Request error in basic info: {str(e)}"
logger.error(error_msg, exc_info=True)
return {
"error": error_msg,
"error_details": {
"type": "RequestException",
"status_code": getattr(e.response, 'status_code', None) if hasattr(e, 'response') else None,
"url": url
}
}
except Exception as e:
logger.error(f"Error getting basic info: {str(e)}", exc_info=True)
return {"error": str(e)}
error_msg = f"Error getting basic info: {str(e)}"
logger.error(error_msg, exc_info=True)
return {
"error": error_msg,
"error_details": {
"type": type(e).__name__,
"traceback": str(e.__traceback__)
}
}
def _check_ssl(self, domain: str) -> Dict:
"""Check SSL/TLS certificate information."""
logger.debug(f"Checking SSL for {domain}")
scraping_logger.debug(f"Checking SSL for {domain}")
try:
context = ssl.create_default_context()
with socket.create_connection((domain, 443)) as sock:
@@ -190,7 +259,7 @@ class WebsiteAnalyzer:
def _check_dns(self, domain: str) -> Dict:
"""Check DNS records."""
logger.debug(f"Checking DNS for {domain}")
scraping_logger.debug(f"Checking DNS for {domain}")
try:
records = {}
for record_type in ['A', 'AAAA', 'MX', 'NS', 'TXT']:
@@ -200,7 +269,7 @@ class WebsiteAnalyzer:
except dns.resolver.NoAnswer:
records[record_type] = []
except Exception as e:
logger.warning(f"Error resolving {record_type} record: {str(e)}")
scraping_logger.warning(f"Error resolving {record_type} record: {str(e)}")
records[record_type] = []
return records
except Exception as e:
@@ -209,6 +278,7 @@ class WebsiteAnalyzer:
def _get_whois_info(self, domain: str) -> Dict:
"""Get WHOIS information for a domain."""
scraping_logger.debug(f"Getting WHOIS info for {domain}")
try:
w = whois.whois(domain)
@@ -240,7 +310,7 @@ class WebsiteAnalyzer:
def _analyze_content(self, url: str) -> Dict:
"""Analyze website content."""
logger.debug(f"Analyzing content for {url}")
scraping_logger.debug(f"Analyzing content for {url}")
try:
response = self.session.get(url, timeout=10)
response.raise_for_status()
@@ -255,6 +325,14 @@ class WebsiteAnalyzer:
# Count headings
headings = soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])
heading_counts = {
'h1': len(soup.find_all('h1')),
'h2': len(soup.find_all('h2')),
'h3': len(soup.find_all('h3')),
'h4': len(soup.find_all('h4')),
'h5': len(soup.find_all('h5')),
'h6': len(soup.find_all('h6'))
}
# Count images
images = soup.find_all('img')
@@ -262,22 +340,52 @@ class WebsiteAnalyzer:
# Count links
links = soup.find_all('a')
# Count paragraphs
paragraphs = soup.find_all('p')
return {
"word_count": word_count,
"heading_count": len(headings),
"heading_structure": heading_counts,
"image_count": len(images),
"link_count": len(links),
"paragraph_count": len(paragraphs),
"has_meta_description": bool(self._get_meta_description(soup)),
"has_robots_txt": bool(self._get_robots_txt(url)),
"has_sitemap": bool(self._get_sitemap(url))
}
except requests.exceptions.RequestException as e:
logger.error(f"Request error in content analysis: {str(e)}", exc_info=True)
return {
"word_count": 0,
"heading_count": 0,
"heading_structure": {'h1': 0, 'h2': 0, 'h3': 0, 'h4': 0, 'h5': 0, 'h6': 0},
"image_count": 0,
"link_count": 0,
"paragraph_count": 0,
"has_meta_description": False,
"has_robots_txt": False,
"has_sitemap": False,
"error": str(e)
}
except Exception as e:
logger.error(f"Content analysis error: {str(e)}", exc_info=True)
return {"error": str(e)}
return {
"word_count": 0,
"heading_count": 0,
"heading_structure": {'h1': 0, 'h2': 0, 'h3': 0, 'h4': 0, 'h5': 0, 'h6': 0},
"image_count": 0,
"link_count": 0,
"paragraph_count": 0,
"has_meta_description": False,
"has_robots_txt": False,
"has_sitemap": False,
"error": str(e)
}
def _check_performance(self, url: str) -> Dict:
"""Check website performance metrics."""
logger.debug(f"Checking performance for {url}")
scraping_logger.debug(f"Checking performance for {url}")
try:
start_time = datetime.now()
response = self.session.get(url, timeout=10)
@@ -289,11 +397,29 @@ class WebsiteAnalyzer:
"load_time": load_time,
"status_code": response.status_code,
"content_length": len(response.content),
"headers": dict(response.headers)
"headers": dict(response.headers),
"response_time": response.elapsed.total_seconds()
}
except requests.exceptions.RequestException as e:
logger.error(f"Request error in performance check: {str(e)}", exc_info=True)
return {
"load_time": 0,
"status_code": 0,
"content_length": 0,
"headers": {},
"response_time": 0,
"error": str(e)
}
except Exception as e:
logger.error(f"Performance check error: {str(e)}", exc_info=True)
return {"error": str(e)}
return {
"load_time": 0,
"status_code": 0,
"content_length": 0,
"headers": {},
"response_time": 0,
"error": str(e)
}
def _get_meta_description(self, soup: BeautifulSoup) -> Optional[str]:
"""Extract meta description from HTML."""
@@ -308,7 +434,7 @@ class WebsiteAnalyzer:
if response.status_code == 200:
return response.text
except Exception as e:
logger.warning(f"Error fetching robots.txt: {str(e)}")
scraping_logger.warning(f"Error fetching robots.txt: {str(e)}")
return None
def _get_sitemap(self, url: str) -> Optional[str]:
@@ -319,5 +445,253 @@ class WebsiteAnalyzer:
if response.status_code == 200:
return response.text
except Exception as e:
logger.warning(f"Error fetching sitemap.xml: {str(e)}")
return None
scraping_logger.warning(f"Error fetching sitemap.xml: {str(e)}")
return None
def _analyze_seo(self, url: str) -> Dict:
"""Analyze website SEO."""
try:
# Extract content
content, soup, extract_errors = self._extract_content(url)
if not content or not soup:
return {
"error": "Failed to extract content",
"error_details": {"errors": extract_errors}
}
# Analyze meta tags
meta_analysis = self._analyze_meta_tags(soup)
# Analyze content with AI
content_analysis, recommendations = self._analyze_content_with_ai(content)
# Calculate overall score
meta_score = sum([
1 if meta_analysis.title['status'] == 'good' else 0,
1 if meta_analysis.description['status'] == 'good' else 0,
1 if meta_analysis.keywords['status'] == 'good' else 0,
1 if meta_analysis.has_robots else 0,
1 if meta_analysis.has_sitemap else 0
]) * 20 # Scale to 100
overall_score = (
meta_score * 0.3 + # 30% weight for meta tags
content_analysis.readability_score * 0.3 + # 30% weight for readability
content_analysis.content_quality_score * 0.4 # 40% weight for content quality
)
return {
"overall_score": overall_score,
"meta_tags": meta_analysis.__dict__,
"content": content_analysis.__dict__,
"recommendations": [rec.__dict__ for rec in recommendations]
}
except Exception as e:
error_msg = f"Error in SEO analysis: {str(e)}"
logger.error(error_msg, exc_info=True)
return {
"error": error_msg,
"error_details": {
"type": type(e).__name__,
"traceback": str(e.__traceback__)
}
}
def _extract_content(self, url: str) -> Tuple[Optional[str], Optional[BeautifulSoup], List[str]]:
"""Extract content from URL."""
errors = []
try:
response = self.session.get(url, timeout=10)
response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser')
return response.text, soup, errors
except requests.RequestException as e:
error_msg = f"Error fetching URL: {str(e)}"
logger.error(error_msg)
errors.append(error_msg)
return None, None, errors
def _analyze_meta_tags(self, soup: BeautifulSoup) -> MetaTagAnalysis:
"""Analyze meta tags using BeautifulSoup."""
# Title analysis
title = soup.title.string if soup.title else ""
title_analysis = {
'status': 'good' if title and 30 <= len(title) <= 60 else 'needs_improvement',
'value': title,
'recommendation': '' if title and 30 <= len(title) <= 60 else 'Title should be between 30-60 characters'
}
# Meta description analysis
meta_desc = soup.find('meta', attrs={'name': 'description'})
desc = meta_desc.get('content', '') if meta_desc else ""
desc_analysis = {
'status': 'good' if desc and 120 <= len(desc) <= 160 else 'needs_improvement',
'value': desc,
'recommendation': '' if desc and 120 <= len(desc) <= 160 else 'Description should be between 120-160 characters'
}
# Keywords analysis
meta_keywords = soup.find('meta', attrs={'name': 'keywords'})
keywords = meta_keywords.get('content', '') if meta_keywords else ""
keywords_analysis = {
'status': 'good' if keywords else 'needs_improvement',
'value': keywords,
'recommendation': '' if keywords else 'Add relevant keywords meta tag'
}
return MetaTagAnalysis(
title=title_analysis,
description=desc_analysis,
keywords=keywords_analysis,
has_robots=bool(soup.find('meta', attrs={'name': 'robots'})),
has_sitemap=bool(soup.find('link', attrs={'rel': 'sitemap'}))
)
def _analyze_content_with_ai(self, content: str) -> Tuple[ContentAnalysis, List[SEORecommendation]]:
"""Analyze content using AI."""
try:
# Prepare prompt for content analysis
prompt = f"""Analyze the following webpage content for SEO and provide a structured analysis:
Content: {content[:4000]}... # Truncate to avoid token limits
Provide analysis in the following format:
1. Word count
2. Heading structure analysis
3. Keyword density for main topics
4. Readability score (0-100)
5. Content quality score (0-100)
6. List of SEO recommendations with priority (high/medium/low), category, issue, recommendation, and impact
Format the response as JSON."""
try:
# Get AI analysis using llm_text_gen
analysis = llm_text_gen(
prompt=prompt,
system_prompt="You are an SEO expert analyzing website content.",
response_format="json_object"
)
if not analysis:
logger.error("Empty response from AI analysis")
return self._get_fallback_analysis(content)
# Create ContentAnalysis object
content_analysis = ContentAnalysis(
word_count=len(content.split()),
headings_structure=analysis.get('heading_structure', {}),
keyword_density=analysis.get('keyword_density', {}),
readability_score=analysis.get('readability_score', 0),
content_quality_score=analysis.get('content_quality_score', 0)
)
# Create recommendations
recommendations = [
SEORecommendation(
priority=rec['priority'],
category=rec['category'],
issue=rec['issue'],
recommendation=rec['recommendation'],
impact=rec['impact']
)
for rec in analysis.get('recommendations', [])
]
return content_analysis, recommendations
except Exception as e:
logger.error(f"Error in AI analysis: {str(e)}")
return self._get_fallback_analysis(content)
except Exception as e:
logger.error(f"Error in AI analysis setup: {str(e)}")
return self._get_fallback_analysis(content)
def _get_fallback_analysis(self, content: str) -> Tuple[ContentAnalysis, List[SEORecommendation]]:
"""Provide fallback analysis when AI analysis is not available."""
try:
# Basic content analysis
words = content.split()
word_count = len(words)
# Simple readability score based on word count
readability_score = min(100, max(0, word_count / 10))
# Basic content quality score
content_quality_score = min(100, max(0, word_count / 20))
# Create basic recommendations
recommendations = [
SEORecommendation(
priority="high",
category="content",
issue="AI analysis unavailable",
recommendation="Consider running the analysis again with a valid API key for more detailed insights",
impact="Limited analysis capabilities"
)
]
return ContentAnalysis(
word_count=word_count,
headings_structure={},
keyword_density={},
readability_score=readability_score,
content_quality_score=content_quality_score
), recommendations
except Exception as e:
logger.error(f"Error in fallback analysis: {str(e)}")
return ContentAnalysis(
word_count=0,
headings_structure={},
keyword_density={},
readability_score=0,
content_quality_score=0
), []
def analyze_website(url: str) -> Dict:
"""
Analyze a website and return comprehensive results.
Args:
url (str): The URL to analyze
Returns:
Dict: Analysis results including various metrics and checks
"""
logger.info(f"Starting website analysis for URL: {url}")
try:
analyzer = WebsiteAnalyzer()
results = analyzer.analyze_website(url)
# Add success status to results
if "error" in results:
error_msg = f"Error in base analysis: {results['error']}"
logger.error(error_msg)
logger.error(f"Error details: {json.dumps(results.get('error_details', {}), indent=2)}")
return {
"success": False,
"error": error_msg,
"error_details": results.get("error_details", {})
}
# Add success status and wrap results
logger.info("Analysis completed successfully")
logger.debug(f"Analysis results: {json.dumps(results, indent=2)}")
return {
"success": True,
"data": results
}
except Exception as e:
error_msg = f"Error in analyze_website: {str(e)}"
logger.error(error_msg, exc_info=True)
return {
"success": False,
"error": error_msg,
"error_details": {
"type": type(e).__name__,
"traceback": str(e.__traceback__)
}
}

134
lib/utils/website_analyzer/content_gap_analyzer.py Normal file
View File

@@ -0,0 +1,134 @@
from typing import Dict
import json
class ContentGapAnalyzer:
def __init__(self, analyzer):
self.analyzer = analyzer
def analyze(self, url: str) -> Dict:
"""
Analyze content gaps for a given URL.
Args:
url (str): The URL to analyze
Returns:
Dict: Analysis results including content gaps and recommendations
"""
try:
# Get base analysis
logger.info(f"Starting content gap analysis for URL: {url}")
base_analysis = self.analyzer.analyze_website(url)
# Check for errors in base analysis
if not base_analysis.get("success", False):
error_msg = base_analysis.get("error", "Unknown error in website analysis")
error_details = base_analysis.get("error_details", {})
logger.error(f"Base analysis failed: {error_msg}")
logger.error(f"Error details: {json.dumps(error_details, indent=2)}")
return {
"success": False,
"error": error_msg,
"error_details": error_details,
"stage": "base_analysis"
}
# Extract required sections
analysis_data = base_analysis.get("data", {}).get("analysis", {})
required_sections = ["content_info", "basic_info", "performance"]
missing_sections = [section for section in required_sections if section not in analysis_data]
if missing_sections:
error_msg = f"Missing required analysis sections: {', '.join(missing_sections)}"
logger.error(error_msg)
logger.error(f"Available sections: {list(analysis_data.keys())}")
return {
"success": False,
"error": error_msg,
"error_details": {
"missing_sections": missing_sections,
"available_sections": list(analysis_data.keys())
},
"stage": "section_validation"
}
# Extract content metrics
try:
content_info = analysis_data["content_info"]
basic_info = analysis_data["basic_info"]
performance = analysis_data["performance"]
except KeyError as e:
error_msg = f"Error extracting analysis section: {str(e)}"
logger.error(error_msg)
return {
"success": False,
"error": error_msg,
"error_details": {
"type": "KeyError",
"missing_key": str(e),
"available_keys": list(analysis_data.keys())
},
"stage": "data_extraction"
}
# Analyze content gaps
try:
gaps = self._analyze_content_gaps(content_info, basic_info, performance)
except Exception as e:
error_msg = f"Error analyzing content gaps: {str(e)}"
logger.error(error_msg, exc_info=True)
return {
"success": False,
"error": error_msg,
"error_details": {
"type": type(e).__name__,
"traceback": str(e.__traceback__)
},
"stage": "gap_analysis"
}
# Generate recommendations
try:
recommendations = self._generate_recommendations(gaps)
except Exception as e:
error_msg = f"Error generating recommendations: {str(e)}"
logger.error(error_msg, exc_info=True)
return {
"success": False,
"error": error_msg,
"error_details": {
"type": type(e).__name__,
"traceback": str(e.__traceback__)
},
"stage": "recommendation_generation"
}
return {
"success": True,
"data": {
"content_gaps": gaps,
"recommendations": recommendations,
"metrics": {
"word_count": content_info.get("word_count", 0),
"heading_count": content_info.get("heading_count", 0),
"image_count": content_info.get("image_count", 0),
"link_count": content_info.get("link_count", 0),
"paragraph_count": content_info.get("paragraph_count", 0),
"load_time": performance.get("load_time", 0),
"response_time": performance.get("response_time", 0)
}
}
}
except Exception as e:
error_msg = f"Error in content gap analysis: {str(e)}"
logger.error(error_msg, exc_info=True)
return {
"success": False,
"error": error_msg,
"error_details": {
"type": type(e).__name__,
"traceback": str(e.__traceback__)
},
"stage": "general"
}

233
lib/utils/website_analyzer/seo_analyzer.py
View File

@@ -1,233 +0,0 @@
"""SEO analyzer module with AI integration."""
import requests
from bs4 import BeautifulSoup
from datetime import datetime
from typing import Dict, List, Tuple, Optional
from urllib.parse import urlparse
import openai
from loguru import logger
import os
from dotenv import load_dotenv
from .models import (
SEOAnalysisResult,
MetaTagAnalysis,
ContentAnalysis,
SEORecommendation
)
def extract_content(url: str) -> Tuple[Optional[str], Optional[BeautifulSoup], List[str]]:
"""Extract content from URL."""
errors = []
try:
response = requests.get(url, timeout=10)
response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser')
return response.text, soup, errors
except requests.RequestException as e:
error_msg = f"Error fetching URL: {str(e)}"
logger.error(error_msg)
errors.append(error_msg)
return None, None, errors
def analyze_meta_tags(soup: BeautifulSoup) -> MetaTagAnalysis:
"""Analyze meta tags using BeautifulSoup."""
# Title analysis
title = soup.title.string if soup.title else ""
title_analysis = {
'status': 'good' if title and 30 <= len(title) <= 60 else 'needs_improvement',
'value': title,
'recommendation': '' if title and 30 <= len(title) <= 60 else 'Title should be between 30-60 characters'
}
# Meta description analysis
meta_desc = soup.find('meta', attrs={'name': 'description'})
desc = meta_desc.get('content', '') if meta_desc else ""
desc_analysis = {
'status': 'good' if desc and 120 <= len(desc) <= 160 else 'needs_improvement',
'value': desc,
'recommendation': '' if desc and 120 <= len(desc) <= 160 else 'Description should be between 120-160 characters'
}
# Keywords analysis
meta_keywords = soup.find('meta', attrs={'name': 'keywords'})
keywords = meta_keywords.get('content', '') if meta_keywords else ""
keywords_analysis = {
'status': 'good' if keywords else 'needs_improvement',
'value': keywords,
'recommendation': '' if keywords else 'Add relevant keywords meta tag'
}
return MetaTagAnalysis(
title=title_analysis,
description=desc_analysis,
keywords=keywords_analysis,
has_robots=bool(soup.find('meta', attrs={'name': 'robots'})),
has_sitemap=bool(soup.find('link', attrs={'rel': 'sitemap'}))
)
def analyze_content_with_ai(content: str) -> Tuple[ContentAnalysis, List[SEORecommendation]]:
"""Analyze content using AI."""
try:
# Load environment variables
load_dotenv()
# Get API key from environment
api_key = os.getenv('OPENAI_API_KEY')
if not api_key:
raise ValueError("OpenAI API key not found in environment variables")
# Initialize OpenAI client
client = openai.OpenAI(api_key=api_key)
# Prepare prompt for content analysis
prompt = f"""Analyze the following webpage content for SEO and provide a structured analysis:
Content: {content[:4000]}... # Truncate to avoid token limits
Provide analysis in the following format:
1. Word count
2. Heading structure analysis
3. Keyword density for main topics
4. Readability score (0-100)
5. Content quality score (0-100)
6. List of SEO recommendations with priority (high/medium/low), category, issue, recommendation, and impact
Format the response as JSON."""
# Get AI analysis
response = client.chat.completions.create(
model="gpt-4",
messages=[
{"role": "system", "content": "You are an SEO expert analyzing website content."},
{"role": "user", "content": prompt}
],
response_format={"type": "json_object"}
)
# Parse AI response
analysis = response.choices[0].message.content
# Create ContentAnalysis object
content_analysis = ContentAnalysis(
word_count=len(content.split()),
headings_structure=analysis.get('heading_structure', {}),
keyword_density=analysis.get('keyword_density', {}),
readability_score=analysis.get('readability_score', 0),
content_quality_score=analysis.get('content_quality_score', 0)
)
# Create recommendations
recommendations = [
SEORecommendation(
priority=rec['priority'],
category=rec['category'],
issue=rec['issue'],
recommendation=rec['recommendation'],
impact=rec['impact']
)
for rec in analysis.get('recommendations', [])
]
return content_analysis, recommendations
except Exception as e:
logger.error(f"Error in AI analysis: {str(e)}")
return ContentAnalysis(
word_count=len(content.split()),
headings_structure={},
keyword_density={},
readability_score=0,
content_quality_score=0
), []
def analyze_seo(url: str) -> SEOAnalysisResult:
"""Main function to analyze website SEO."""
errors = []
warnings = []
# Validate URL
try:
parsed_url = urlparse(url)
if not all([parsed_url.scheme, parsed_url.netloc]):
errors.append("Invalid URL format")
raise ValueError("Invalid URL format")
except Exception as e:
errors.append(f"URL parsing error: {str(e)}")
return SEOAnalysisResult(
url=url,
analyzed_at=datetime.now(),
overall_score=0,
meta_tags=None,
content=None,
recommendations=[],
errors=errors,
warnings=warnings,
success=False
)
# Extract content
content, soup, extract_errors = extract_content(url)
errors.extend(extract_errors)
if not content or not soup:
return SEOAnalysisResult(
url=url,
analyzed_at=datetime.now(),
overall_score=0,
meta_tags=None,
content=None,
recommendations=[],
errors=errors,
warnings=warnings,
success=False
)
try:
# Analyze meta tags
meta_analysis = analyze_meta_tags(soup)
# Analyze content with AI
content_analysis, recommendations = analyze_content_with_ai(content)
# Calculate overall score
meta_score = sum([
1 if meta_analysis.title['status'] == 'good' else 0,
1 if meta_analysis.description['status'] == 'good' else 0,
1 if meta_analysis.keywords['status'] == 'good' else 0,
1 if meta_analysis.has_robots else 0,
1 if meta_analysis.has_sitemap else 0
]) * 20 # Scale to 100
overall_score = (
meta_score * 0.3 + # 30% weight for meta tags
content_analysis.readability_score * 0.3 + # 30% weight for readability
content_analysis.content_quality_score * 0.4 # 40% weight for content quality
)
return SEOAnalysisResult(
url=url,
analyzed_at=datetime.now(),
overall_score=overall_score,
meta_tags=meta_analysis,
content=content_analysis,
recommendations=recommendations,
errors=errors,
warnings=warnings,
success=True
)
except Exception as e:
error_msg = f"Error in SEO analysis: {str(e)}"
logger.error(error_msg)
errors.append(error_msg)
return SEOAnalysisResult(
url=url,
analyzed_at=datetime.now(),
overall_score=0,
meta_tags=None,
content=None,
recommendations=[],
errors=errors,
warnings=warnings,
success=False
)