Long form content generation, web researched

.github/workflows/python-package.yml

@@ -1,40 +0,0 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
-
-name: Python package
-
-on:
-  push:
-    branches: [ "main" ]
-  pull_request:
-    branches: [ "main" ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.9", "3.10", "3.11"]
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install flake8 pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Test with pytest
-      run: |
-        pytest

lib/ai_web_researcher/google_serp_search.py

@@ -90,7 +90,7 @@ def google_search(query):
 
 
 
-def perform_serpapi_google_search(query, location="in"):
+def perform_serpapi_google_search(query):
     """
     Perform a Google search using the SerpApi service.
 
@@ -102,6 +102,12 @@ def perform_serpapi_google_search(query, location="in"):
     Returns:
         dict: A dictionary containing the search results.
     """
+    try:
+        logger.info("Reading Web search config values from main_config")
+        geo_location, search_language, num_results, time_range, include_domains, similar_url = read_return_config_section('web_research')
+    except Exception as err:
+        logger.error(f"Failed to read web research params: {err}")
+        return
     try:
         # Check if API key is provided
         if not os.getenv("SERPAPI_KEY"):
@@ -158,10 +164,10 @@ def perform_serperdev_google_search(query):
     payload = json.dumps({
         "q": query,
         "gl": geo_loc,
+        "location": geo_loc,
         "hl": lang,
         "num": num_results,
         "autocorrect": True,
-        "page": 1,
         "type": "search",
         "engine": "google"
     })

lib/ai_writers/keywords_to_blog.py

@@ -45,15 +45,17 @@ def write_blog_from_keywords(search_keywords, url=None):
     # logger.info/check the final blog content.
     logger.info("\n######### Draft1: Finished Blog from Google web search: ###########\n\n")
 
+    # TBD: The 3 drafts are very similar, the intention was to fact check the AI content multiple times.
+    # Commenting it out for blog writing, using Tavily for other forms of writing.
     # Do Tavily AI research to augument the above blog.
-    try:
-        tavily_search_result, t_titles = do_tavily_ai_search(search_keywords)
-        example_blog_titles.append(t_titles)
-        blog_markdown_str = blog_with_research(blog_markdown_str, tavily_search_result)
-        logger.info(f"######### Blog content after Tavily AI research: ######### \n\n{blog_markdown_str}\n\n")
-    except Exception as err:
-        logger.error(f"Failed to do Tavily AI research: {err}")
-    logger.info("######### Draft2: Blog content after Tavily AI research: #########\n\n")
+    #try:
+    #    tavily_search_result, t_titles = do_tavily_ai_search(search_keywords)
+    #    example_blog_titles.append(t_titles)
+    #    blog_markdown_str = blog_with_research(blog_markdown_str, tavily_search_result)
+    #    logger.info(f"######### Blog content after Tavily AI research: ######### \n\n{blog_markdown_str}\n\n")
+    #except Exception as err:
+    #    logger.error(f"Failed to do Tavily AI research: {err}")
+    #logger.info("######### Draft2: Blog content after Tavily AI research: #########\n\n")
 
     try:
         # Do Metaphor/Exa AI search.

lib/utils/read_main_config_params.py

@@ -52,6 +52,17 @@ def read_return_config_section(config_section):
 
             return blog_tone, blog_demographic, blog_type, blog_language, blog_output_format
 
+        elif 'web_research' in config_section:
+            # Access the config file and return the specified values
+            geo_location = config.get('web_research', 'geo_location')
+            search_language = config.get('web_research', 'search_language')
+            num_results = config.getint('web_research', 'num_results')
+            time_range = config.get('web_research', 'time_range')
+            include_domains = config.get('web_research', 'include_domains')
+            similar_url = config.get('web_research', 'similar_url')
+
+            return geo_location, search_language, num_results, time_range, include_domains, similar_url
+
     except FileNotFoundError:
         logger.error(f"Configuration file not found: {config_path}")
         raise