WIP- Try AI-Writer and Web research; working. Working on usuability aspects.

lib/ai_web_researcher/google_serp_search.py

@@ -66,23 +66,23 @@ def google_search(query):
     Returns:
         list: List of search results based on the specified flag.
     """
-    try:
-        perform_serpapi_google_search(query)
-        logger.info(f"FIXME: Google serapi: {query}")
-        #return process_search_results(search_result)
-    except Exception as err:
-        logger.error(f"ERROR: Check Here: https://serpapi.com/. Your requests may be over. {err}")
+    #try:
+    #    perform_serpapi_google_search(query)
+    #    logger.info(f"FIXME: Google serapi: {query}")
+    #    #return process_search_results(search_result)
+    #except Exception as err:
+    #    logger.error(f"ERROR: Check Here: https://serpapi.com/. Your requests may be over. {err}")
 
     # Retry with serper.dev
     try:
         logger.info("Trying Google search with Serper.dev: https://serper.dev/api-key")
         search_result = perform_serperdev_google_search(query)
         process_search_results(search_result)
+        return(search_result)
     except Exception as err:
         logger.error(f"Failed to do Google search with serper.dev: {err}")
 
-    return(search_result)
-    
+ 
 #    # Retry with BROWSERLESS API
 #    try:
 #        search_result = perform_browserless_google_search(query)
@@ -118,7 +118,10 @@ def perform_serpapi_google_search(query, location="in"):
     try:
         # Check if API key is provided
         if not os.getenv("SERPAPI_KEY"):
-            raise ValueError("SERPAPI_KEY key is required for SerpApi")
+            #raise ValueError("SERPAPI_KEY key is required for SerpApi")
+            logger.error("SERPAPI_KEY key is required for SerpApi")
+            return
+            
 
         # Create a GoogleSearch instance
         search = GoogleSearch({
@@ -164,7 +167,7 @@ def perform_serperdev_google_search(query):
         "q": query,
         "gl": "in",
         "hl": "en",
-        "num": 5,
+        "num": 10,
         "autocorrect": True,
         "page": 1,
         "type": "search",

lib/ai_web_researcher/google_trends_researcher.py

@@ -23,6 +23,8 @@ Note: Ensure that the required libraries are installed using 'pip install pytren
 """
 
 import os
+import time # I wish
+import random
 import requests
 import numpy as np
 import sys
@@ -186,6 +188,7 @@ def get_related_topics_and_save_csv(search_keywords):
             data = pytrends.related_topics()
         except Exception as err:
             logger.error(f"Failed to get pytrends realted topics: {err}")
+            return
         # Extract data from the result
         top_topics = list(data.values())[0]['top']
         rising_topics = list(data.values())[0]['rising']
@@ -238,6 +241,8 @@ def get_results(query):
     try:
         query = urllib.parse.quote_plus(query)
         response = get_source(f"https://suggestqueries.google.com/complete/search?output=chrome&hl=en&q={query}")
+        time.sleep(random.uniform(0.1, 0.6))
+
         if response:
             response.raise_for_status()
             results = json.loads(response.text)
@@ -501,6 +506,8 @@ def do_google_trends_analysis(search_term):
             else:
                 all_the_keywords.append(suggestions_df['Keywords'].tolist())
             all_the_keywords = ','.join([', '.join(filter(None, map(str, sublist))) for sublist in all_the_keywords])
+            # Generate a random sleep time between 2 and 3 seconds 
+            time.sleep(random.uniform(2, 3))
 
 #        
 #        # FIXME: Get result from vision GPT. Fetch and visualize Google Trends data
@@ -510,12 +517,16 @@ def do_google_trends_analysis(search_term):
 #        result_df = plot_interest_by_region(search_term)
 #        
         # Display additional information
-        result_df = get_related_topics_and_save_csv(search_term)
-        # Extract 'Top' topic_title
-        top_topic_title = result_df['topic_title'].values.tolist()
-        # Join each sublist into one string separated by comma
-        #top_topic_title = [','.join(filter(None, map(str, sublist))) for sublist in top_topic_title]
-        top_topic_title = ','.join([', '.join(filter(None, map(str, sublist))) for sublist in top_topic_title])
+        try:
+            result_df = get_related_topics_and_save_csv(search_term)
+            # Extract 'Top' topic_title
+            if result_df:
+                top_topic_title = result_df['topic_title'].values.tolist()
+                # Join each sublist into one string separated by comma
+                #top_topic_title = [','.join(filter(None, map(str, sublist))) for sublist in top_topic_title]
+                top_topic_title = ','.join([', '.join(filter(None, map(str, sublist))) for sublist in top_topic_title])
+        except Exception as err:
+            logger.error(f"Failed to get results from google trends related topics: {err}")
 
         # TBD: Not getting great results OR unable to understand them.
         #all_the_keywords += top_topic_title

lib/ai_web_researcher/gpt_online_researcher.py

@@ -9,7 +9,6 @@ import json
 from pathlib import Path
 import sys
 from typing import List, NamedTuple
-from loguru import logger
 from datetime import datetime
 
 from ..gpt_providers.gemini_pro_text import gemini_text_response
@@ -17,8 +16,9 @@ from .tavily_ai_search import get_tavilyai_results
 from .metaphor_basic_neural_web_search import metaphor_find_similar, metaphor_search_articles
 from .google_serp_search import google_search
 from .google_trends_researcher import do_google_trends_analysis
-from .web_research_report import write_web_research_report
+#from .web_research_report import write_web_research_report
 
+from loguru import logger
 # Configure logger
 logger.remove()
 logger.add(sys.stdout,

lib/ai_web_researcher/tavily_ai_search.py

@@ -66,7 +66,7 @@ def get_tavilyai_results(keywords, include_urls, search_depth="advanced"):
     # Retrieve API keys
     api_key = os.getenv('TAVILY_API_KEY')
     if not api_key:
-        raise ValueError("API keys for Tavily or OpenAI are not set.")
+        raise ValueError("API keys for Tavily is Not set.")
 
     # Initialize Tavily client
     try:

lib/ai_web_researcher/web_research_report.py

@@ -1,11 +1,10 @@
-from langchain.adapters.openai import convert_openai_messages
-from langchain.chat_models import ChatOpenAI
-
+import os
 from ..gpt_providers.gemini_pro_text import gemini_text_response
 
 
-def write_web_research_report(web_research, faq_questions, gpt_provider="gemini"):
+def write_web_research_report(web_research, faq_questions):
     """ """
+    gpt_provider = os.environ["GPT_PROVIDER"]
     if "gemini" in gpt_provider:
             prompt = ["You are an SEO and marketing expert, who writes unique, factual and comprehensive research reports."
                     "I will provide you web research report as json data and a list of related FAQ questions."