ALwrity Version 0.5.1 (Fastapi + React)
This commit is contained in:
ajaysi
340
ToBeMigrated/ai_seo_tools/sitemap_analysis.py
Normal file
340
ToBeMigrated/ai_seo_tools/sitemap_analysis.py
Normal file
@@ -0,0 +1,340 @@
|
||||
import streamlit as st
|
||||
import advertools as adv
|
||||
import pandas as pd
|
||||
import plotly.graph_objects as go
|
||||
from urllib.error import URLError
|
||||
import xml.etree.ElementTree as ET
|
||||
import requests
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
Main function to run the Sitemap Analyzer Streamlit app.
|
||||
"""
|
||||
st.title("📊 Sitemap Analyzer")
|
||||
st.write("""
|
||||
This tool analyzes a website's sitemap to understand its content structure and publishing trends.
|
||||
Enter a sitemap URL to start your analysis.
|
||||
""")
|
||||
|
||||
sitemap_url = st.text_input(
|
||||
"Please enter the sitemap URL:",
|
||||
"https://www.example.com/sitemap.xml"
|
||||
)
|
||||
|
||||
if st.button("Analyze Sitemap"):
|
||||
try:
|
||||
sitemap_df = fetch_all_sitemaps(sitemap_url)
|
||||
if sitemap_df is not None and not sitemap_df.empty:
|
||||
sitemap_df = process_lastmod_column(sitemap_df)
|
||||
ppmonth = analyze_content_trends(sitemap_df)
|
||||
sitemap_df = categorize_and_shorten_sitemaps(sitemap_df)
|
||||
|
||||
display_key_metrics(sitemap_df, ppmonth)
|
||||
plot_sitemap_content_distribution(sitemap_df)
|
||||
plot_content_trends(ppmonth)
|
||||
plot_content_type_breakdown(sitemap_df)
|
||||
plot_publishing_frequency(sitemap_df)
|
||||
|
||||
st.success("🎉 Analysis complete!")
|
||||
else:
|
||||
st.error("No valid URLs found in the sitemap.")
|
||||
except URLError as e:
|
||||
st.error(f"Error fetching the sitemap: {e}")
|
||||
except Exception as e:
|
||||
st.error(f"An unexpected error occurred: {e}")
|
||||
|
||||
|
||||
def fetch_all_sitemaps(sitemap_url):
|
||||
"""
|
||||
Fetches all sitemaps from the provided sitemap URL and concatenates their URLs into a DataFrame.
|
||||
|
||||
Parameters:
|
||||
sitemap_url (str): The URL of the sitemap.
|
||||
|
||||
Returns:
|
||||
DataFrame: A DataFrame containing all URLs from the sitemaps.
|
||||
"""
|
||||
st.write(f"🚀 Fetching and analyzing the sitemap: {sitemap_url}...")
|
||||
|
||||
try:
|
||||
sitemap_df = fetch_sitemap(sitemap_url)
|
||||
|
||||
if sitemap_df is not None:
|
||||
all_sitemaps = sitemap_df.loc[
|
||||
sitemap_df['loc'].str.contains('sitemap'),
|
||||
'loc'
|
||||
].tolist()
|
||||
|
||||
if all_sitemaps:
|
||||
st.write(
|
||||
f"🔄 Found {len(all_sitemaps)} additional sitemaps. Fetching data from them..."
|
||||
)
|
||||
all_urls_df = pd.DataFrame()
|
||||
|
||||
for sitemap in all_sitemaps:
|
||||
try:
|
||||
st.write(f"Fetching URLs from {sitemap}...")
|
||||
temp_df = fetch_sitemap(sitemap)
|
||||
if temp_df is not None:
|
||||
all_urls_df = pd.concat(
|
||||
[all_urls_df, temp_df], ignore_index=True
|
||||
)
|
||||
except Exception as e:
|
||||
st.error(f"Error fetching {sitemap}: {e}")
|
||||
|
||||
st.write(
|
||||
f"✅ Successfully fetched {len(all_urls_df)} URLs from all sitemaps."
|
||||
)
|
||||
return all_urls_df
|
||||
|
||||
else:
|
||||
st.write(f"✅ Successfully fetched {len(sitemap_df)} URLs from the main sitemap.")
|
||||
return sitemap_df
|
||||
else:
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
st.error(f"⚠️ Error fetching the sitemap: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def fetch_sitemap(url):
|
||||
"""
|
||||
Fetches and parses the sitemap from the provided URL.
|
||||
|
||||
Parameters:
|
||||
url (str): The URL of the sitemap.
|
||||
|
||||
Returns:
|
||||
DataFrame: A DataFrame containing the URLs from the sitemap.
|
||||
"""
|
||||
try:
|
||||
response = requests.get(url)
|
||||
response.raise_for_status()
|
||||
|
||||
ET.fromstring(response.content)
|
||||
|
||||
sitemap_df = adv.sitemap_to_df(url)
|
||||
return sitemap_df
|
||||
|
||||
except requests.RequestException as e:
|
||||
st.error(f"⚠️ Request error: {e}")
|
||||
return None
|
||||
except ET.ParseError as e:
|
||||
st.error(f"⚠️ XML parsing error: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def process_lastmod_column(sitemap_df):
|
||||
"""
|
||||
Processes the 'lastmod' column in the sitemap DataFrame by converting it to DateTime format and setting it as the index.
|
||||
|
||||
Parameters:
|
||||
sitemap_df (DataFrame): The sitemap DataFrame.
|
||||
|
||||
Returns:
|
||||
DataFrame: The processed sitemap DataFrame with 'lastmod' as the index.
|
||||
"""
|
||||
st.write("📅 Converting 'lastmod' column to DateTime format and setting it as the index...")
|
||||
|
||||
try:
|
||||
sitemap_df = sitemap_df.dropna(subset=['lastmod'])
|
||||
sitemap_df['lastmod'] = pd.to_datetime(sitemap_df['lastmod'])
|
||||
sitemap_df.set_index('lastmod', inplace=True)
|
||||
|
||||
st.write("✅ 'lastmod' column successfully converted to DateTime format and set as the index.")
|
||||
return sitemap_df
|
||||
|
||||
except Exception as e:
|
||||
st.error(f"⚠️ Error processing the 'lastmod' column: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def categorize_and_shorten_sitemaps(sitemap_df):
|
||||
"""
|
||||
Categorizes and shortens the sitemap names in the sitemap DataFrame.
|
||||
|
||||
Parameters:
|
||||
sitemap_df (DataFrame): The sitemap DataFrame.
|
||||
|
||||
Returns:
|
||||
DataFrame: The sitemap DataFrame with categorized and shortened sitemap names.
|
||||
"""
|
||||
st.write("🔍 Categorizing and shortening sitemap names...")
|
||||
|
||||
try:
|
||||
sitemap_df['sitemap_name'] = sitemap_df['sitemap'].str.split('/').str[4]
|
||||
sitemap_df['sitemap_name'] = sitemap_df['sitemap_name'].replace({
|
||||
'sitemap-site-kasko-fiyatlari.xml': 'Kasko',
|
||||
'sitemap-site-bireysel.xml': 'Personal',
|
||||
'sitemap-site-kurumsal.xml': 'Cooperate',
|
||||
'sitemap-site-arac-sigortasi.xml': 'Car',
|
||||
'sitemap-site.xml': 'Others'
|
||||
})
|
||||
|
||||
st.write("✅ Sitemap names categorized and shortened.")
|
||||
return sitemap_df
|
||||
|
||||
except Exception as e:
|
||||
st.error(f"⚠️ Error categorizing sitemap names: {e}")
|
||||
return sitemap_df
|
||||
|
||||
|
||||
def analyze_content_trends(sitemap_df):
|
||||
"""
|
||||
Analyzes content publishing trends in the sitemap DataFrame.
|
||||
|
||||
Parameters:
|
||||
sitemap_df (DataFrame): The sitemap DataFrame.
|
||||
|
||||
Returns:
|
||||
Series: A Series representing the number of contents published each month.
|
||||
"""
|
||||
st.write("📅 Analyzing content publishing trends...")
|
||||
|
||||
try:
|
||||
ppmonth = sitemap_df.resample('M').size()
|
||||
sitemap_df['monthly_count'] = sitemap_df.index.to_period('M').value_counts().sort_index()
|
||||
|
||||
st.write("✅ Content trends analysis completed.")
|
||||
return ppmonth
|
||||
|
||||
except Exception as e:
|
||||
st.error(f"⚠️ Error during content trends analysis: {e}")
|
||||
return pd.Series()
|
||||
|
||||
|
||||
def display_key_metrics(sitemap_df, ppmonth):
|
||||
"""
|
||||
Displays key metrics of the sitemap analysis.
|
||||
|
||||
Parameters:
|
||||
sitemap_df (DataFrame): The sitemap DataFrame.
|
||||
ppmonth (Series): The Series representing the number of contents published each month.
|
||||
"""
|
||||
st.write("### Key Metrics")
|
||||
|
||||
total_urls = len(sitemap_df)
|
||||
total_articles = ppmonth.sum()
|
||||
average_frequency = ppmonth.mean()
|
||||
|
||||
st.write(f"**Total URLs Found:** {total_urls:,}")
|
||||
st.write(f"**Total Articles Published:** {total_articles:,}")
|
||||
st.write(f"**Average Monthly Publishing Frequency:** {average_frequency:.2f} articles/month")
|
||||
|
||||
|
||||
def plot_sitemap_content_distribution(sitemap_df):
|
||||
"""
|
||||
Plots the content distribution by sitemap categories.
|
||||
|
||||
Parameters:
|
||||
sitemap_df (DataFrame): The sitemap DataFrame.
|
||||
"""
|
||||
st.write("📊 Visualizing content amount by sitemap categories...")
|
||||
|
||||
try:
|
||||
if 'sitemap_name' in sitemap_df.columns:
|
||||
stmc = sitemap_df.groupby('sitemap_name').size()
|
||||
fig = go.Figure()
|
||||
fig.add_bar(x=stmc.index, y=stmc.values, name='Sitemap Categories')
|
||||
fig.update_layout(
|
||||
title='Content Amount by Sitemap Categories',
|
||||
xaxis_title='Sitemap Categories',
|
||||
yaxis_title='Number of Articles',
|
||||
paper_bgcolor='#E5ECF6'
|
||||
)
|
||||
st.plotly_chart(fig)
|
||||
else:
|
||||
st.warning("⚠️ The 'sitemap_name' column is missing in the data.")
|
||||
|
||||
except Exception as e:
|
||||
st.error(f"⚠️ Error during sitemap content distribution plotting: {e}")
|
||||
|
||||
|
||||
def plot_content_trends(ppmonth):
|
||||
"""
|
||||
Plots the content publishing trends over time.
|
||||
|
||||
Parameters:
|
||||
ppmonth (Series): The Series representing the number of contents published each month.
|
||||
"""
|
||||
st.write("📈 Plotting content publishing trends over time...")
|
||||
|
||||
try:
|
||||
fig = go.Figure()
|
||||
fig.add_scatter(x=ppmonth.index, y=ppmonth.values, mode='lines+markers', name='Publishing Trends')
|
||||
fig.update_layout(
|
||||
title='Content Publishing Trends Over Time',
|
||||
xaxis_title='Month',
|
||||
yaxis_title='Number of Articles',
|
||||
paper_bgcolor='#E5ECF6'
|
||||
)
|
||||
st.plotly_chart(fig)
|
||||
|
||||
except Exception as e:
|
||||
st.error(f"⚠️ Error during content trends plotting: {e}")
|
||||
|
||||
|
||||
def plot_content_type_breakdown(sitemap_df):
|
||||
"""
|
||||
Plots the content type breakdown.
|
||||
|
||||
Parameters:
|
||||
sitemap_df (DataFrame): The sitemap DataFrame.
|
||||
"""
|
||||
st.write("🔍 Plotting content type breakdown...")
|
||||
|
||||
try:
|
||||
if 'sitemap_name' in sitemap_df.columns and not sitemap_df['sitemap_name'].empty:
|
||||
content_type_counts = sitemap_df['sitemap_name'].value_counts()
|
||||
st.write("Content Type Counts:", content_type_counts)
|
||||
|
||||
if not content_type_counts.empty:
|
||||
fig = go.Figure(data=[go.Pie(labels=content_type_counts.index, values=content_type_counts.values)])
|
||||
fig.update_layout(
|
||||
title='Content Type Breakdown',
|
||||
paper_bgcolor='#E5ECF6'
|
||||
)
|
||||
st.plotly_chart(fig)
|
||||
else:
|
||||
st.warning("⚠️ No content types to display.")
|
||||
else:
|
||||
st.warning("⚠️ The 'sitemap_name' column is missing or empty.")
|
||||
|
||||
except Exception as e:
|
||||
st.error(f"⚠️ Error during content type breakdown plotting: {e}")
|
||||
|
||||
|
||||
def plot_publishing_frequency(sitemap_df):
|
||||
"""
|
||||
Plots the publishing frequency by month.
|
||||
|
||||
Parameters:
|
||||
sitemap_df (DataFrame): The sitemap DataFrame.
|
||||
"""
|
||||
st.write("📆 Plotting publishing frequency by month...")
|
||||
|
||||
try:
|
||||
if not sitemap_df.empty:
|
||||
frequency_by_month = sitemap_df.index.to_period('M').value_counts().sort_index()
|
||||
frequency_by_month.index = frequency_by_month.index.astype(str)
|
||||
|
||||
fig = go.Figure()
|
||||
fig.add_bar(x=frequency_by_month.index, y=frequency_by_month.values, name='Publishing Frequency')
|
||||
fig.update_layout(
|
||||
title='Publishing Frequency by Month',
|
||||
xaxis_title='Month',
|
||||
yaxis_title='Number of Articles',
|
||||
paper_bgcolor='#E5ECF6'
|
||||
)
|
||||
st.plotly_chart(fig)
|
||||
else:
|
||||
st.warning("⚠️ No data available to plot publishing frequency.")
|
||||
|
||||
except Exception as e:
|
||||
st.error(f"⚠️ Error during publishing frequency plotting: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user