From 1c7ba95b271b0289f5dc25f6ad03c9b68903545a Mon Sep 17 00:00:00 2001 From: ajaysi Date: Wed, 24 Apr 2024 08:58:09 +0530 Subject: [PATCH] Long form content generation, web researched --- .github/workflows/python-package.yml | 40 --------------------- lib/ai_web_researcher/google_serp_search.py | 10 ++++-- lib/ai_writers/keywords_to_blog.py | 18 +++++----- lib/utils/read_main_config_params.py | 11 ++++++ 4 files changed, 29 insertions(+), 50 deletions(-) delete mode 100644 .github/workflows/python-package.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml deleted file mode 100644 index fd04aae9..00000000 --- a/.github/workflows/python-package.yml +++ /dev/null @@ -1,40 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python - -name: Python package - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11"] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest diff --git a/lib/ai_web_researcher/google_serp_search.py b/lib/ai_web_researcher/google_serp_search.py index fecf278b..c3d9df99 100644 --- a/lib/ai_web_researcher/google_serp_search.py +++ b/lib/ai_web_researcher/google_serp_search.py @@ -90,7 +90,7 @@ def google_search(query): -def perform_serpapi_google_search(query, location="in"): +def perform_serpapi_google_search(query): """ Perform a Google search using the SerpApi service. @@ -102,6 +102,12 @@ def perform_serpapi_google_search(query, location="in"): Returns: dict: A dictionary containing the search results. """ + try: + logger.info("Reading Web search config values from main_config") + geo_location, search_language, num_results, time_range, include_domains, similar_url = read_return_config_section('web_research') + except Exception as err: + logger.error(f"Failed to read web research params: {err}") + return try: # Check if API key is provided if not os.getenv("SERPAPI_KEY"): @@ -158,10 +164,10 @@ def perform_serperdev_google_search(query): payload = json.dumps({ "q": query, "gl": geo_loc, + "location": geo_loc, "hl": lang, "num": num_results, "autocorrect": True, - "page": 1, "type": "search", "engine": "google" }) diff --git a/lib/ai_writers/keywords_to_blog.py b/lib/ai_writers/keywords_to_blog.py index 1728c6d6..2723df74 100644 --- a/lib/ai_writers/keywords_to_blog.py +++ b/lib/ai_writers/keywords_to_blog.py @@ -45,15 +45,17 @@ def write_blog_from_keywords(search_keywords, url=None): # logger.info/check the final blog content. logger.info("\n######### Draft1: Finished Blog from Google web search: ###########\n\n") + # TBD: The 3 drafts are very similar, the intention was to fact check the AI content multiple times. + # Commenting it out for blog writing, using Tavily for other forms of writing. # Do Tavily AI research to augument the above blog. - try: - tavily_search_result, t_titles = do_tavily_ai_search(search_keywords) - example_blog_titles.append(t_titles) - blog_markdown_str = blog_with_research(blog_markdown_str, tavily_search_result) - logger.info(f"######### Blog content after Tavily AI research: ######### \n\n{blog_markdown_str}\n\n") - except Exception as err: - logger.error(f"Failed to do Tavily AI research: {err}") - logger.info("######### Draft2: Blog content after Tavily AI research: #########\n\n") + #try: + # tavily_search_result, t_titles = do_tavily_ai_search(search_keywords) + # example_blog_titles.append(t_titles) + # blog_markdown_str = blog_with_research(blog_markdown_str, tavily_search_result) + # logger.info(f"######### Blog content after Tavily AI research: ######### \n\n{blog_markdown_str}\n\n") + #except Exception as err: + # logger.error(f"Failed to do Tavily AI research: {err}") + #logger.info("######### Draft2: Blog content after Tavily AI research: #########\n\n") try: # Do Metaphor/Exa AI search. diff --git a/lib/utils/read_main_config_params.py b/lib/utils/read_main_config_params.py index bdfb0c1c..241b6c8d 100644 --- a/lib/utils/read_main_config_params.py +++ b/lib/utils/read_main_config_params.py @@ -52,6 +52,17 @@ def read_return_config_section(config_section): return blog_tone, blog_demographic, blog_type, blog_language, blog_output_format + elif 'web_research' in config_section: + # Access the config file and return the specified values + geo_location = config.get('web_research', 'geo_location') + search_language = config.get('web_research', 'search_language') + num_results = config.getint('web_research', 'num_results') + time_range = config.get('web_research', 'time_range') + include_domains = config.get('web_research', 'include_domains') + similar_url = config.get('web_research', 'similar_url') + + return geo_location, search_language, num_results, time_range, include_domains, similar_url + except FileNotFoundError: logger.error(f"Configuration file not found: {config_path}") raise