revert to 93075dc

alwrity.py

@@ -3,24 +3,16 @@ import os
 import json
 import base64
 from datetime import datetime
+import streamlit as st
 from lib.utils.environment_utils import load_environment
 from lib.utils.config_manager import save_config
+from lib.utils.ui_setup import setup_ui
 from lib.utils.api_key_manager import check_api_keys, check_llm_environs
-from lib.utils.content_generators import ai_writers, content_planning_tools, blog_from_keyword, story_input_section, essay_writer, ai_news_writer, ai_finance_ta_writer, write_ai_prod_desc, competitor_analysis, ai_agents_content_planner
+from lib.utils.content_generators import ai_writers, content_planning_tools, blog_from_keyword, story_input_section, essay_writer, ai_news_writer, ai_finance_ta_writer, write_ai_prod_desc, do_web_research, competitor_analysis, ai_agents_content_planner
 from lib.utils.seo_tools import ai_seo_tools
 from lib.utils.alwrity_utils import ai_agents_team, ai_social_writer
 from lib.utils.file_processor import load_image, read_prompts, write_prompts
 from lib.utils.voice_processing import record_voice
-from lib.ai_web_researcher.google_trends_researcher import (
-    fetch_multirange_interest_over_time,
-    fetch_historical_hourly_interest,
-    fetch_trending_searches,
-    fetch_realtime_search_trends,
-    fetch_top_charts,
-    fetch_suggestions
-)
-import pandas as pd
-import matplotlib.pyplot as plt
 
 # Placeholder function definitions for missing functions
 def blog_from_audio():
@@ -103,16 +95,15 @@ def check_llm_environs():
     return True
 
 
-def save_config(config, save_now=False):
+def save_config(config):
     """
     Saves the provided configuration dictionary to a JSON file specified by the environment variable.
     """
-    if save_now:
-        try:
-            with open(os.getenv("ALWRITY_CONFIG"), "w") as config_file:
-                json.dump(config, config_file, indent=4)
-        except Exception as e:
-            st.error(f"An error occurred while saving the configuration: {e}")
+    try:
+        with open(os.getenv("ALWRITY_CONFIG"), "w") as config_file:
+            json.dump(config, config_file, indent=4)
+    except Exception as e:
+        st.error(f"An error occurred while saving the configuration: {e}")
 
 
 # Sidebar configuration
@@ -285,9 +276,8 @@ def sidebar_configuration():
         }
     }
 
-    # Option to save the configuration explicitly
-    if st.sidebar.button("Save Configuration"):
-        save_config(config, save_now=True)
+    # Writing the configuration to a file whenever a change is made
+    save_config(config)
 
 
 
@@ -443,60 +433,7 @@ def content_planning_tools():
     choice = st.radio("Select a content planning tool:", options, index=0, format_func=lambda x: f"🔍 {x}")
     
     if choice == "Keywords Researcher":
-        st.title("Web Research Assistant")
-        st.write("Enter keywords for web research. The keywords should be at least three words long.")
-        
-        search_keywords = st.text_input("Search Keywords", placeholder="Enter keywords for web research...")
-        if st.button("Start Web Research"):
-            if search_keywords and len(search_keywords.split()) >= 3:
-                try:
-                    st.info(f"Starting web research on given keywords: {search_keywords}")
-                    with st.spinner("Performing web research..."):
-                        # Fetch and display multirange interest over time
-                        st.subheader("Multirange Interest Over Time")
-                        multirange_data = fetch_multirange_interest_over_time([search_keywords], ['today 3-m', 'today 1-m'])
-                        st.dataframe(multirange_data)
-
-                        # Fetch and display historical hourly interest
-                        st.subheader("Historical Hourly Interest")
-                        hourly_data = fetch_historical_hourly_interest([search_keywords], '2023-01-01', '2023-01-31')
-                        st.dataframe(hourly_data)
-
-                        # Fetch and display trending searches
-                        st.subheader("Trending Searches")
-                        trending_data = fetch_trending_searches()
-                        st.dataframe(trending_data)
-
-                        # Fetch and display realtime search trends
-                        st.subheader("Realtime Search Trends")
-                        realtime_data = fetch_realtime_search_trends()
-                        st.dataframe(realtime_data)
-
-                        # Fetch and display top charts
-                        st.subheader("Top Charts")
-                        top_charts_data = fetch_top_charts(2023)
-                        st.dataframe(top_charts_data)
-
-                        # Fetch and display suggestions
-                        st.subheader("Suggestions")
-                        suggestions = fetch_suggestions(search_keywords)
-                        st.dataframe(pd.DataFrame(suggestions))
-
-                        # Example of plotting with Matplotlib
-                        st.subheader("Interest Over Time Plot")
-                        plt.figure(figsize=(10, 6))
-                        plt.plot(multirange_data['date'], multirange_data[search_keywords], label=search_keywords)
-                        plt.title(f'Interest Over Time for "{search_keywords}"')
-                        plt.xlabel('Date')
-                        plt.ylabel('Interest')
-                        plt.legend()
-                        st.pyplot(plt)
-
-                    st.success("Web research completed successfully!")
-                except Exception as err:
-                    st.error(f"ERROR: Failed to do web research: {err}")
-            else:
-                st.warning("Search keywords should be at least three words long. Please try again.")
+        do_web_research()
     elif choice == "Competitor Analysis":
         competitor_analysis()
     elif choice == "Content Calender Ideator":

lib/ai_marketing_tools/ai_backlinking.py

@@ -1,308 +0,0 @@
-#Problem:
-#
-#Finding websites for guest posts is manual, tedious, and time-consuming. Communicating with webmasters, maintaining conversations, and keeping track of backlinking opportunities is difficult to scale. Content creators and marketers struggle with discovering new websites and consistently getting backlinks.
-#Solution:
-#
-#An AI-powered backlinking app that automates web research, scrapes websites, extracts contact information, and sends personalized outreach emails to webmasters. This would simplify the entire process, allowing marketers to scale their backlinking strategy with minimal manual intervention.
-#Core Workflow:
-#
-#    User Input:
-#        Keyword Search: The user inputs a keyword (e.g., "AI writers").
-#        Search Queries: Your app will append various search strings to this keyword to find backlinking opportunities (e.g., "AI writers + 'Write for Us'").
-#
-#    Web Research:
-#
-#        Use search engines or web scraping to run multiple queries:
-#            Keyword + "Guest Contributor"
-#            Keyword + "Add Guest Post"
-#            Keyword + "Write for Us", etc.
-#
-#        Collect URLs of websites that have pages or posts related to guest post opportunities.
-#
-#    Scrape Website Data:
-#        Contact Information Extraction:
-#            Scrape the website for contact details (email addresses, contact forms, etc.).
-#            Use natural language processing (NLP) to understand the type of content on the website and who the contact person might be (webmaster, editor, or guest post manager).
-#        Website Content Understanding:
-#            Scrape a summary of each website’s content (e.g., their blog topics, categories, and tone) to personalize the email based on the site's focus.
-#
-#    Personalized Outreach:
-#        AI Email Composition:
-#            Compose personalized outreach emails based on:
-#                The scraped data (website content, topic focus, etc.).
-#                The user's input (what kind of guest post or content they want to contribute).
-#            Example: “Hi [Webmaster Name], I noticed that your site [Site Name] features high-quality content about [Topic]. I would love to contribute a guest post on [Proposed Topic] in exchange for a backlink.”
-#
-#    Automated Email Sending:
-#        Review Emails (Optional HITL):
-#            Let users review and approve the personalized emails before they are sent, or allow full automation.
-#        Send Emails:
-#            Automate email dispatch through an integrated SMTP or API (e.g., Gmail API, SendGrid).
-#            Keep track of which emails were sent, bounced, or received replies.
-#
-#    Scaling the Search:
-#        Repeat for Multiple Keywords:
-#            Run the same scraping and outreach process for a list of relevant keywords, either automatically suggested or uploaded by the user.
-#        Keep Track of Sent Emails:
-#            Maintain a log of all sent emails, responses, and follow-up reminders to avoid repetition or forgotten leads.
-#
-#    Tracking Responses and Follow-ups:
-#        Automated Responses:
-#            If a website replies positively, AI can respond with predefined follow-up emails (e.g., proposing topics, confirming submission deadlines).
-#        Follow-up Reminders:
-#            If there’s no reply, the system can send polite follow-up reminders at pre-set intervals.
-#
-#Key Features:
-#
-#    Automated Web Scraping:
-#        Scrape websites for guest post opportunities using a predefined set of search queries based on user input.
-#        Extract key information like email addresses, names, and submission guidelines.
-#
-#    Personalized Email Writing:
-#        Leverage AI to create personalized emails using the scraped website information.
-#        Tailor each email to the tone, content style, and focus of the website.
-#
-#    Email Sending Automation:
-#        Integrate with email platforms (e.g., Gmail, SendGrid, or custom SMTP).
-#        Send automated outreach emails with the ability for users to review first (HITL - Human-in-the-loop) or automate completely.
-#
-#    Customizable Email Templates:
-#        Allow users to customize or choose from a set of email templates for different types of outreach (e.g., guest post requests, follow-up emails, submission offers).
-#
-#    Lead Tracking and Management:
-#        Track all emails sent, monitor replies, and keep track of successful backlinks.
-#        Log each lead’s status (e.g., emailed, responded, no reply) to manage future interactions.
-#
-#    Multiple Keywords/Queries:
-#        Allow users to run the same process for a batch of keywords, automatically generating relevant search queries for each.
-#
-#    AI-Driven Follow-Up:
-#        Schedule follow-up emails if there is no response after a specified period.
-#
-#    Reports and Analytics:
-#        Provide users with reports on how many emails were sent, opened, replied to, and successful backlink placements.
-#
-#Advanced Features (for Scaling and Optimization):
-#
-#    Domain Authority Filtering:
-#        Use SEO APIs (e.g., Moz, Ahrefs) to filter websites based on their domain authority or backlink strength.
-#        Prioritize high-authority websites to maximize the impact of backlinks.
-#
-#    Spam Detection:
-#        Use AI to detect and avoid spammy or low-quality websites that might harm the user’s SEO.
-#
-#    Contact Form Auto-Fill:
-#        If the site only offers a contact form (without email), automatically fill and submit the form with AI-generated content.
-#
-#    Dynamic Content Suggestions:
-#        Suggest guest post topics based on the website’s focus, using NLP to analyze the site's existing content.
-#
-#    Bulk Email Support:
-#        Allow users to bulk-send outreach emails while still personalizing each message for scalability.
-#
-#    AI Copy Optimization:
-#        Use copywriting AI to optimize email content, adjusting tone and CTA based on the target audience.
-#
-#Challenges and Considerations:
-#
-#    Legal Compliance:
-#        Ensure compliance with anti-spam laws (e.g., CAN-SPAM, GDPR) by including unsubscribe options or manual email approval.
-#
-#    Scraping Limits:
-#        Be mindful of scraping limits on certain websites and employ smart throttling or use API-based scraping for better reliability.
-#
-#    Deliverability:
-#        Ensure emails are delivered properly without landing in spam folders by integrating proper email authentication (SPF, DKIM) and using high-reputation SMTP servers.
-#
-#    Maintaining Email Personalization:
-#        Striking the balance between automating the email process and keeping each message personal enough to avoid being flagged as spam.
-#
-#Technology Stack:
-#
-#    Web Scraping: BeautifulSoup, Scrapy, or Puppeteer for scraping guest post opportunities and contact information.
-#    Email Automation: Integrate with Gmail API, SendGrid, or Mailgun for sending emails.
-#    NLP for Personalization: GPT-based models for email generation and web content understanding.
-#    Frontend: React or Vue for the user interface.
-#    Backend: Python/Node.js with Flask or Express for the API and automation logic.
-#    Database: MongoDB or PostgreSQL to track leads, emails, and responses.
-#
-#This solution will significantly streamline the backlinking process by automating the most tedious tasks, from finding sites to personalizing outreach, enabling marketers to focus on content creation and high-level strategies.
-
-from lib.ai_web_researcher.firecrawl_web_crawler import scrape_website
-from lib.gpt_providers.text_generation.main_text_generation import llm_text_gen
-from lib.ai_web_researcher.firecrawl_web_crawler import scrape_url
-
-
-def generate_search_queries(keyword):
-    """
-    Generate a list of search queries for finding guest post opportunities.
-    
-    Args:
-        keyword (str): The keyword to base the search queries on.
-    
-    Returns:
-        list: A list of search queries.
-    """
-    search_queries = [
-        f"{keyword} + 'Guest Contributor'",
-        f"{keyword} + 'Add Guest Post'",
-        f"{keyword} + 'Guest Bloggers Wanted'",
-        f"{keyword} + 'Guest Posts Roundup'",
-        f"{keyword} + 'Write for Us'",
-        f"{keyword} + 'Submit Guest Post'",
-        f"{keyword} + 'Submit a Guest Article'",
-        f"{keyword} + 'Guest Bloggers Wanted'",
-        f"{keyword} + 'Submit an article'",
-        f"{keyword} + 'Suggest a guest post'",
-        f"{keyword} + 'Send a guest post'",
-        f"{keyword} + 'Become a Guest Blogger'",
-        f"{keyword} + 'guest post opportunities'",
-        f"{keyword} + 'this is a guest post by'",
-        f"{keyword} + 'This post was written by'",
-        f"{keyword} + 'guest post courtesy of'",
-        f"{keyword} + 'submit article'"
-    ]
-    return search_queries
-
-def find_backlink_opportunities(keyword):
-    """
-    Find backlink opportunities by scraping websites based on search queries.
-    
-    Args:
-        keyword (str): The keyword to search for backlink opportunities.
-    
-    Returns:
-        list: A list of results from the scraped websites.
-    """
-    search_queries = generate_search_queries(keyword)
-    results = []
-    for query in search_queries:
-        # Placeholder for a function to search and get URLs
-        urls = search_for_urls(query)
-        for url in urls:
-            website_data = scrape_website(url)
-            if website_data:
-                contact_info = extract_contact_info(url)
-                # Construct a prompt for the LLM
-                prompt = f"""
-                Analyze the following website content and provide insights:
-                Content: {website_data.get("content_summary", "")}
-
-                Please provide:
-                1. A brief summary of what the website is about.
-                2. Guidelines to follow for guest posting.
-                3. Suggested topics to write on.
-                4. Any other insights to help make a highly personalized reach out and decision making.
-                """
-
-                insights = llm_text_gen(prompt)
-
-                detailed_result = {
-                    "url": url,
-                    "metadata": {
-                        "title": website_data.get("metadata", {}).get("title", ""),
-                        "description": website_data.get("metadata", {}).get("description", ""),
-                        "keywords": website_data.get("metadata", {}).get("keywords", []),
-                        "author": website_data.get("metadata", {}).get("author", ""),
-                    },
-                    "content_summary": website_data.get("content_summary", ""),
-                    "contact_info": contact_info,
-                    "insights": insights,
-                    "backlink_opportunity": {
-                        "query": query,
-                        "context": "Guest post opportunity"
-                    }
-                }
-                results.append(detailed_result)
-    return results
-
-def compose_personalized_email(website_data, insights, user_proposal):
-    """
-    Compose a personalized outreach email using AI LLM based on website data, insights, and user proposal.
-
-    Args:
-        website_data (dict): The data of the website including metadata and contact info.
-        insights (str): Insights generated by the LLM about the website.
-        user_proposal (str): The user's proposal for a guest post or content contribution.
-
-    Returns:
-        str: A personalized email message.
-    """
-    contact_name = website_data.get("contact_info", {}).get("name", "Webmaster")
-    site_name = website_data.get("metadata", {}).get("title", "your site")
-    proposed_topic = user_proposal.get("topic", "a guest post")
-
-    # Construct a prompt for the LLM to generate a personalized email
-    prompt = f"""
-    You are an AI assistant tasked with composing a personalized outreach email. 
-    Use the following details to craft a compelling message:
-
-    Contact Name: {contact_name}
-    Website Name: {site_name}
-    Proposed Topic: {proposed_topic}
-    Insights: {insights}
-
-    The email should be professional, engaging, and tailored to the recipient's interests and the website's focus.
-    """
-
-    # Generate the email using the LLM
-    email_body = llm_text_gen(prompt)
-
-    return email_body
-
-
-
-def search_for_urls(query):
-    """
-    Search for URLs based on a query using Firecrawl.
-
-    Args:
-        query (str): The search query.
-
-    Returns:
-        list: A list of URLs.
-    """
-    # Use Firecrawl to perform the search
-    result = scrape_url(query)
-    if result and 'urls' in result:
-        return result['urls']
-    return []
-
-
-from lib.ai_web_researcher.firecrawl_web_crawler import extract_data
-
-def extract_contact_info(url):
-    """
-    Extract contact information from a website using Firecrawl's LLM Extract feature.
-    
-    Args:
-        url (str): The URL of the website to extract contact information from.
-    
-    Returns:
-        dict: Extracted contact information.
-    """
-    schema = {
-        "type": "object",
-        "properties": {
-            "emails": {
-                "type": "array",
-                "items": {
-                    "type": "string",
-                    "format": "email"
-                }
-            },
-            "contact_forms": {
-                "type": "array",
-                "items": {
-                    "type": "string",
-                    "format": "uri"
-                }
-            }
-        },
-        "required": ["emails", "contact_forms"]
-    }
-    
-    result = extract_data(url, schema)
-    if result and 'extract' in result:
-        return result['extract']
-    return {}

lib/ai_marketing_tools/backlinking_ui_streamlit.py

@@ -1,60 +0,0 @@
-import streamlit as st
-import pandas as pd
-from streamlit_aggrid import AgGrid, GridOptionsBuilder, GridUpdateMode
-from lib.ai_marketing_tools.ai_backlinking import find_backlink_opportunities, compose_personalized_email
-
-
-# Streamlit UI function
-def backlinking_ui():
-    st.title("AI Backlinking Tool")
-
-    # Step 1: Get user inputs
-    keyword = st.text_input("Enter a keyword", value="technology")
-
-    # Step 2: Generate backlink opportunities
-    if st.button("Find Backlink Opportunities"):
-        if keyword:
-            backlink_opportunities = find_backlink_opportunities(keyword)
-
-            # Convert results to a DataFrame for display
-            df = pd.DataFrame(backlink_opportunities)
-
-            # Create a selectable table using st-aggrid
-            gb = GridOptionsBuilder.from_dataframe(df)
-            gb.configure_selection('multiple', use_checkbox=True, groupSelectsChildren=True)
-            gridOptions = gb.build()
-
-            grid_response = AgGrid(
-                df,
-                gridOptions=gridOptions,
-                update_mode=GridUpdateMode.SELECTION_CHANGED,
-                height=200,
-                width='100%'
-            )
-
-            selected_rows = grid_response['selected_rows']
-
-            if selected_rows:
-                st.write("Selected Opportunities:")
-                st.table(pd.DataFrame(selected_rows))
-
-                # Step 3: Option to generate personalized emails for selected opportunities
-                if st.button("Generate Emails for Selected Opportunities"):
-                    user_proposal = {
-                        "user_name": st.text_input("Your Name", value="John Doe"),
-                        "user_email": st.text_input("Your Email", value="john@example.com")
-                    }
-
-                    emails = []
-                    for selected in selected_rows:
-                        insights = f"Insights based on content from {selected['url']}."
-                        email = compose_personalized_email(selected, insights, user_proposal)
-                        emails.append(email)
-
-                    st.subheader("Generated Emails:")
-                    for email in emails:
-                        st.write(email)
-                        st.markdown("---")
-
-        else:
-            st.error("Please enter a keyword.")

lib/ai_web_researcher/firecrawl_web_crawler.py

@@ -1,8 +1,8 @@
-from firecrawl_client import initialize_client
-import logging
 import os
 from pathlib import Path
+
 from firecrawl import FirecrawlApp
+import logging
 from dotenv import load_dotenv
 # Load environment variables from .env file
 load_dotenv(Path('../../.env'))
@@ -10,6 +10,18 @@ load_dotenv(Path('../../.env'))
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
 
+def initialize_client():
+    """
+    Initialize and return a Firecrawl client.
+
+    Args:
+        api_key (str): Your Firecrawl API key.
+
+    Returns:
+        firecrawl.Client: An instance of the Firecrawl client.
+    """
+    return FirecrawlApp(api_key=os.getenv("FIRECRAWL_API_KEY"))
+
 
 def scrape_website(website_url, depth=1, max_pages=10):
     """

lib/ai_web_researcher/google_trends_researcher.py

@@ -23,7 +23,7 @@ Note: Ensure that the required libraries are installed using 'pip install pytren
 """
 
 import os
-import time
+import time # I wish
 import random
 import requests
 import numpy as np
@@ -45,9 +45,7 @@ from urllib.parse import quote_plus
 from tqdm import tqdm
 from tabulate import tabulate
 from pytrends.request import TrendReq
-from datetime import datetime
 from loguru import logger
-from wordcloud import WordCloud
 
 # Configure logger
 logger.remove()
@@ -57,161 +55,7 @@ logger.add(sys.stdout,
            )
 
 
-def fetch_multirange_interest_over_time(keywords, timeframes):
-    """
-    Fetch multirange interest over time for given keywords and timeframes.
-
-    Args:
-        keywords (list): List of keywords.
-        timeframes (list): List of timeframes.
-
-    Returns:
-        pd.DataFrame: DataFrame containing interest over time data.
-    """
-    try:
-        pytrends = TrendReq(hl='en-US', tz=360)
-        pytrends.build_payload(keywords, timeframe=timeframes)
-        data = pytrends.multirange_interest_over_time()
-        data = data.reset_index()
-
-        # Display data and explanation
-        print(f"\n📈 Multirange Interest Over Time for '{keywords}':")
-        print("This metric shows the interest of each keyword over multiple time ranges, allowing you to see trends and patterns.")
-        print(data.to_string(index=False))
-
-        return data
-    except Exception as e:
-        logger.error(f"Error in fetch_multirange_interest_over_time: {e}")
-        return pd.DataFrame()
-
-
-def fetch_historical_hourly_interest(keywords, start_date, end_date):
-    """
-    Fetch historical hourly interest for given keywords.
-
-    Args:
-        keywords (list): List of keywords.
-        start_date (str): Start date in 'YYYY-MM-DD' format.
-        end_date (str): End date in 'YYYY-MM-DD' format.
-
-    Returns:
-        pd.DataFrame: DataFrame containing historical hourly interest data.
-    """
-    try:
-        pytrends = TrendReq(hl='en-US', tz=360)
-        data = pytrends.get_historical_interest(keywords, year_start=int(start_date[:4]), month_start=int(start_date[5:7]), day_start=int(start_date[8:10]), hour_start=0, year_end=int(end_date[:4]), month_end=int(end_date[5:7]), day_end=int(end_date[8:10]), hour_end=0)
-        data = data.reset_index()
-
-        # Display data and explanation
-        print(f"\n⏰ Historical Hourly Interest for '{keywords}':")
-        print("This metric provides the interest level of each keyword on an hourly basis, useful for understanding daily patterns.")
-        print(data.to_string(index=False))
-
-        return data
-    except Exception as e:
-        logger.error(f"Error in fetch_historical_hourly_interest: {e}")
-        return pd.DataFrame()
-
-
-def fetch_trending_searches(region='united_states'):
-    """
-    Fetch trending searches for a given region.
-
-    Args:
-        region (str): Region for which to fetch trending searches.
-
-    Returns:
-        pd.DataFrame: DataFrame containing trending searches.
-    """
-    try:
-        pytrends = TrendReq(hl='en-US', tz=360)
-        data = pytrends.trending_searches(pn=region)
-
-        # Display data and explanation
-        print(f"\n🔥 Trending Searches in '{region}':")
-        print("These are the searches that are currently trending in the specified region, indicating popular topics.")
-        print(data.to_string(index=False))
-
-        return data
-    except Exception as e:
-        logger.error(f"Error in fetch_trending_searches: {e}")
-        return pd.DataFrame()
-
-
-def fetch_realtime_search_trends(region='US'):
-    """
-    Fetch realtime search trends for a given region.
-
-    Args:
-        region (str): Region for which to fetch realtime search trends.
-
-    Returns:
-        pd.DataFrame: DataFrame containing realtime search trends.
-    """
-    try:
-        pytrends = TrendReq(hl='en-US', tz=360)
-        data = pytrends.realtime_trending_searches(pn=region)
-
-        # Display data and explanation
-        print(f"\n📊 Realtime Search Trends in '{region}':")
-        print("These are the searches that are trending in real-time, providing insights into current events and interests.")
-        print(data.to_string(index=False))
-
-        return data
-    except Exception as e:
-        logger.error(f"Error in fetch_realtime_search_trends: {e}")
-        return pd.DataFrame()
-
-
-def fetch_top_charts(year, region='GLOBAL'):
-    """
-    Fetch top charts for a given year and region.
-
-    Args:
-        year (int): Year for which to fetch top charts.
-        region (str): Region for which to fetch top charts.
-
-    Returns:
-        pd.DataFrame: DataFrame containing top charts.
-    """
-    try:
-        pytrends = TrendReq(hl='en-US', tz=360)
-        data = pytrends.top_charts(year, geo=region)
-
-        # Display data and explanation
-        print(f"\n🏆 Top Charts for {year} in '{region}':")
-        print("These charts show the top searches for a given year and region, highlighting significant trends over time.")
-        print(data.to_string(index=False))
-
-        return data
-    except Exception as e:
-        logger.error(f"Error in fetch_top_charts: {e}")
-        return pd.DataFrame()
-
-
-def fetch_suggestions(keyword):
-    """
-    Fetch suggestions for a given keyword.
-
-    Args:
-        keyword (str): Keyword for which to fetch suggestions.
-
-    Returns:
-        list: List of suggestions.
-    """
-    try:
-        pytrends = TrendReq(hl='en-US', tz=360)
-        suggestions = pytrends.suggestions(keyword)
-
-        # Display data and explanation
-        print(f"\n💡 Suggestions for '{keyword}':")
-        print("These are suggested search terms related to the given keyword, useful for expanding your search strategy.")
-        print(pd.DataFrame(suggestions).to_string(index=False))
-
-        return suggestions
-    except Exception as e:
-        logger.error(f"Error in fetch_suggestions: {e}")
-        return []
+def fetch_google_trends_interest_overtime(keyword):
     try:
         pytrends = TrendReq(hl='en-US', tz=360)
         pytrends.build_payload([keyword], timeframe='today 1-y', geo='US')
@@ -345,11 +189,6 @@ def get_related_topics_and_save_csv(search_keywords):
             logger.error(f"Failed to get pytrends realted topics: {err}")
             return None
 
-        # Check if data contains expected keys
-        if not data or 'top' not in list(data.values())[0] or 'rising' not in list(data.values())[0]:
-            logger.error("No related topics found.")
-            return pd.DataFrame()
-
         # Extract data from the result
         top_topics = list(data.values())[0]['top']
         rising_topics = list(data.values())[0]['rising']
@@ -658,7 +497,7 @@ def do_google_trends_analysis(search_term):
     all_the_keywords = []
     try:
         for asearch_term in search_term:
-            # FIXME: Lets work with a single root keyword.
+            #FIXME: Lets work with a single root keyword.
             suggestions_df = get_suggestions_for_keyword(asearch_term)
             if len(suggestions_df['Keywords']) > 10:
                 result_df = perform_keyword_clustering(suggestions_df)
@@ -671,14 +510,13 @@ def do_google_trends_analysis(search_term):
             # Generate a random sleep time between 2 and 3 seconds 
             time.sleep(random.uniform(2, 3))
 
-        # Fetch and display various Google Trends data
-        fetch_multirange_interest_over_time(search_term, ['today 3-m', 'today 1-m'])
-        fetch_historical_hourly_interest(search_term, '2023-01-01', '2023-01-31')
-        fetch_trending_searches()
-        fetch_realtime_search_trends()
-        fetch_top_charts(2023)
-        fetch_suggestions(search_term[0])
-
+#        
+#        # FIXME: Get result from vision GPT. Fetch and visualize Google Trends data
+#        #trends_data = fetch_google_trends_interest_overtime("llamaindex")
+#
+#        # FIXME: Plot Interest Over time.
+#        result_df = plot_interest_by_region(search_term)
+#        
         # Display additional information
         try:
             result_df = get_related_topics_and_save_csv(search_term)
@@ -686,10 +524,13 @@ def do_google_trends_analysis(search_term):
             if result_df:
                 top_topic_title = result_df['topic_title'].values.tolist()
                 # Join each sublist into one string separated by comma
+                #top_topic_title = [','.join(filter(None, map(str, sublist))) for sublist in top_topic_title]
                 top_topic_title = ','.join([', '.join(filter(None, map(str, sublist))) for sublist in top_topic_title])
         except Exception as err:
             logger.error(f"Failed to get results from google trends related topics: {err}")
 
+        # TBD: Not getting great results OR unable to understand them.
+        #all_the_keywords += top_topic_title
         all_the_keywords = all_the_keywords.split(',')
         # Split the list into chunks of 5 keywords
         chunk_size = 4
@@ -706,6 +547,7 @@ def do_google_trends_analysis(search_term):
             logger.error(f"Failed to save search results: {save_results_err}")
         print(table)
         
+        #generate_wordcloud(all_the_keywords)
         return(all_the_keywords)
     except Exception as e:
         logger.error(f"Error in Google Trends Analysis: {e}")

lib/utils/alwrity_utils.py

@@ -2,6 +2,7 @@ import re
 import streamlit as st
 import tempfile
 from loguru import logger
+from lib.ai_web_researcher.gpt_online_researcher import gpt_web_researcher
 from lib.ai_web_researcher.metaphor_basic_neural_web_search import metaphor_find_similar
 from lib.ai_writers.keywords_to_blog_streamlit import write_blog_from_keywords
 from lib.ai_writers.speech_to_blog.main_audio_to_blog import generate_audio_blog
@@ -13,6 +14,7 @@ from lib.ai_writers.facebook_ai_writer import facebook_post_writer
 from lib.ai_writers.linkedin_ai_writer import linked_post_writer
 from lib.ai_writers.twitter_ai_writer import tweet_writer 
 from lib.ai_writers.insta_ai_writer import insta_writer
+from lib.ai_writers.youtube_ai_writer import write_yt_title, write_yt_description, write_yt_script
 from lib.ai_writers.web_url_ai_writer import blog_from_url
 from lib.ai_writers.image_ai_writer import blog_from_image
 from lib.ai_writers.ai_essay_writer import ai_essay_generator
@@ -20,6 +22,7 @@ import os
 import PyPDF2
 import tiktoken
 import openai
+from lib.gpt_providers.text_to_image_generation.main_generate_image_from_prompt import generate_image
 from lib.ai_seo_tools.seo_structured_data import ai_structured_data
 from lib.ai_seo_tools.content_title_generator import ai_title_generator
 from lib.ai_seo_tools.meta_desc_generator import metadesc_generator_main
@@ -35,41 +38,41 @@ from ..gpt_providers.text_generation.main_text_generation import llm_text_gen
 
 
 def is_youtube_link(text):
-    """Check if the provided text is a YouTube link."""
-    if text:
+    if text is not None:
         youtube_regex = re.compile(r'(https?://)?(www\.)?(youtube|youtu|youtube-nocookie)\.(com|be)/(watch\?v=|embed/|v/|.+\?v=)?([^&=%\?]{11})')
         return youtube_regex.match(text)
 
 
 def is_web_link(text):
-    """Check if the provided text is a web link."""
-    if text:
+    if text is not None:
         web_regex = re.compile(r'(https?://)?(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)')
         return web_regex.match(text)
 
 
 def process_input(input_text, uploaded_file):
-    """Process the input text or uploaded file and determine its type."""
-    if input_text:
-        if is_youtube_link(input_text):
-            if input_text.startswith("https://www.youtube.com/") or input_text.startswith("http://www.youtube.com/"):
-                return "youtube_url"
-            else:
-                st.error("Invalid YouTube URL. Please enter a valid URL.")
-                return None
-        elif is_web_link(input_text):
-            return "web_url"
+    if input_text and is_youtube_link(input_text):
+        if input_text.startswith("https://www.youtube.com/") or input_text.startswith("http://www.youtube.com/"):
+            return "youtube_url"
         else:
-            return "keywords"
+            st.error("Invalid YouTube URL. Please enter a valid URL.")
+            return None
+
+    elif input_text and is_web_link(input_text):
+        return "web_url"
     
-    if uploaded_file:
+    elif input_text:
+        return "keywords"
+    
+    if uploaded_file is not None:
         file_details = {"filename": uploaded_file.name, "filetype": uploaded_file.type}
         st.write(file_details)
         if uploaded_file.type.startswith("text/"):
             content = uploaded_file.read().decode("utf-8")
             st.text(content)
+
         elif uploaded_file.type == "application/pdf":
             return "PDF_file"
+
         elif uploaded_file.type in ["application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/msword"]:
             st.write("Word document uploaded. Add your DOCX processing logic here.")
         elif uploaded_file.type.startswith("image/"):
@@ -500,6 +503,24 @@ def competitor_analysis():
             st.error("Please enter a valid URL.")
 
 
+def do_web_research():
+    """ Input keywords and do web research and present a report."""
+    st.title("Web Research Assistant")
+    st.write("Enter keywords for web research. The keywords should be at least three words long.")
+    
+    search_keywords = st.text_input("Search Keywords", placeholder="Enter keywords for web research...")
+    if st.button("Start Web Research"):
+        if search_keywords and len(search_keywords.split()) >= 3:
+            try:
+                st.info(f"Starting web research on given keywords: {search_keywords}")
+                with st.spinner("Performing web research..."):
+                    web_research_result = gpt_web_researcher(search_keywords)
+                st.success("Web research completed successfully!")
+                st.write(web_research_result)
+            except Exception as err:
+                st.error(f"ERROR: Failed to do web research: {err}")
+        else:
+            st.warning("Search keywords should be at least three words long. Please try again.")
 
 
 def ai_finance_ta_writer():

lib/utils/content_generators.py

@@ -1,23 +1,11 @@
 import streamlit as st
 from lib.utils.alwrity_utils import (
     blog_from_keyword, ai_agents_team, essay_writer, ai_news_writer, ai_seo_tools,
-    ai_finance_ta_writer, ai_social_writer, competitor_analysis
+    ai_finance_ta_writer, ai_social_writer, do_web_research, competitor_analysis
 )
-import pandas as pd
-import matplotlib.pyplot as plt
 from lib.ai_writers.ai_story_writer.story_writer import story_input_section
-from lib.ai_web_researcher.google_trends_researcher import (
-    fetch_multirange_interest_over_time,
-    fetch_historical_hourly_interest,
-    fetch_trending_searches,
-    fetch_realtime_search_trends,
-    fetch_top_charts,
-    fetch_suggestions
-)
 from lib.ai_writers.ai_product_description_writer import write_ai_prod_desc
 from lib.content_planning_calender.content_planning_agents_alwrity_crew import ai_agents_content_planner
-from pytrends.request import TrendReq
-from datetime import datetime
 
 def ai_writers():
     options = [
@@ -61,64 +49,10 @@ def content_planning_tools():
     choice = st.radio("Select a content planning tool:", options, index=0, format_func=lambda x: f"🔍 {x}")
     
     if choice == "Keywords Researcher":
-        st.title("Web Research Assistant")
-        st.write("Enter keywords for web research. The keywords should be at least three words long.")
-        
-        search_keywords = st.text_input("Search Keywords", placeholder="Enter keywords for web research...")
-        if st.button("Start Web Research"):
-            if search_keywords and len(search_keywords.split()) >= 3:
-                try:
-                    st.info(f"Starting web research on given keywords: {search_keywords}")
-                    with st.spinner("Performing web research..."):
-                        # Fetch and display multirange interest over time
-                        st.subheader("Multirange Interest Over Time")
-                        multirange_data = fetch_multirange_interest_over_time([search_keywords], ['today 3-m', 'today 1-m'])
-                        st.dataframe(multirange_data)
-
-                        # Fetch and display historical hourly interest
-                        st.subheader("Historical Hourly Interest")
-                        hourly_data = fetch_historical_hourly_interest([search_keywords], '2023-01-01', '2023-01-31')
-                        st.dataframe(hourly_data)
-
-                        # Fetch and display trending searches
-                        st.subheader("Trending Searches")
-                        trending_data = fetch_trending_searches()
-                        st.dataframe(trending_data)
-
-                        # Fetch and display realtime search trends
-                        st.subheader("Realtime Search Trends")
-                        realtime_data = fetch_realtime_search_trends()
-                        st.dataframe(realtime_data)
-
-                        # Fetch and display top charts
-                        st.subheader("Top Charts")
-                        top_charts_data = fetch_top_charts(2023)
-                        st.dataframe(top_charts_data)
-
-                        # Fetch and display suggestions
-                        st.subheader("Suggestions")
-                        suggestions = fetch_suggestions(search_keywords)
-                        st.dataframe(pd.DataFrame(suggestions))
-
-                        # Example of plotting with Matplotlib
-                        st.subheader("Interest Over Time Plot")
-                        plt.figure(figsize=(10, 6))
-                        plt.plot(multirange_data['date'], multirange_data[search_keywords], label=search_keywords)
-                        plt.title(f'Interest Over Time for "{search_keywords}"')
-                        plt.xlabel('Date')
-                        plt.ylabel('Interest')
-                        plt.legend()
-                        st.pyplot(plt)
-
-                    st.success("Web research completed successfully!")
-                except Exception as err:
-                    st.error(f"ERROR: Failed to do web research: {err}")
-            else:
-                st.warning("Search keywords should be at least three words long. Please try again.")
-    elif choice == "Keywords Researcher":
-        google_trends_analysis()
-        competitor_analysis()
+        do_web_research()
     elif choice == "Competitor Analysis":
+        competitor_analysis()
+    elif choice == "Content Calender Ideator":
         plan_keywords = st.text_input(
             "**Enter Your main Keywords to get 2 months content calendar:**",
             placeholder="Enter 2-3 main keywords to generate AI content calendar with keyword researched blog titles",
@@ -129,46 +63,3 @@ def content_planning_tools():
                 ai_agents_content_planner(plan_keywords)
             else:
                 st.error("Come on, really, Enter some keywords to plan on..")
-def google_trends_analysis():
-    st.title("Google Trends Analysis")
-
-    # Prompt user for required input
-    keyword = st.text_input("Enter Keyword(s)", help="Enter one or more keywords separated by commas.")
-    
-    # Optional inputs with intelligent defaults
-    start_time = st.date_input("Start Time", value=datetime(2004, 1, 1), help="Start date for the analysis.")
-    end_time = st.date_input("End Time", value=datetime.now(), help="End date for the analysis.")
-    geo = st.text_input("Geographic Location", value="US", help="Location of interest (e.g., 'US').")
-    hl = st.text_input("Preferred Language", value="en", help="Preferred language (e.g., 'en').")
-    timezone = st.number_input("Timezone", value=360, help="Timezone offset in minutes from UTC.")
-    category = st.number_input("Category", value=0, help="Category to search within.")
-    property = st.selectbox("Google Property", options=["", "images", "news", "youtube", "froogle"], help="Google property to filter on.")
-    resolution = st.selectbox("Resolution", options=["COUNTRY", "REGION", "CITY", "DMA"], help="Granularity of the geo search.")
-    granular_time_resolution = st.checkbox("Granular Time Resolution", value=False, help="Use finer time resolution if applicable.")
-
-    if st.button("Analyze"):
-        if not keyword:
-            st.error("Keyword is required.")
-            return
-
-        # Initialize pytrends
-        pytrends = TrendReq(hl=hl, tz=timezone)
-
-        # Build the payload
-        pytrends.build_payload(
-            kw_list=keyword.split(','),
-            timeframe=f"{start_time.strftime('%Y-%m-%d')} {end_time.strftime('%Y-%m-%d')}",
-            geo=geo,
-            cat=category,
-            gprop=property
-        )
-
-        # Fetch interest over time
-        interest_over_time_df = pytrends.interest_over_time()
-        st.subheader("Interest Over Time")
-        st.dataframe(interest_over_time_df)
-
-        # Fetch interest by region
-        interest_by_region_df = pytrends.interest_by_region(resolution=resolution)
-        st.subheader("Interest By Region")
-        st.dataframe(interest_by_region_df)