diff --git a/.env.bk b/.env.bk
new file mode 100644
index 00000000..0f9500c7
--- /dev/null
+++ b/.env.bk
@@ -0,0 +1,23 @@
+# Required should be minimum.
+
+# Model to use, presently supporting Openai, Gemini, ollama(wip), minstral(wip)
+GPT_PROVIDER="google"
+MODEL_TO_USE="gemini"
+
+
+# Provide the key for MODEL_TO_USE
+OPENAI_API_KEY=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
+GEMINI_API_KEY=AIzaSyDM3SfGw9SeBpB39-PWpY8qQjt9OZAeZdM
+MISTRAL_API_KEY=zaObt2UMIjKKx3Vwmt6G8ccIsrVZI4PT
+
+
+#--------------------------------------------------------------------------------
+
+SERPER_API_KEY=281b6a882ae28164c08d0bee7113d63ed9f5e547
+TAVILY_API_KEY=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
+METAPHOR_API_KEY=94105da2-2954-4b49-933c-1920c7a1a2b3
+
+#---------------------------------------------------------------------------------
+
+# Tools and utilities APIs.
+TINIFY_API_KEY=crJLXQL3DMdXYHz1LvZy60Rd1vYH8mvZ
diff --git a/.gitignore b/.gitignore
index a4a37578..b075aca7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,6 @@
*.pyc
__pycache__
-pseo-experiemnts/
+pseo-experiments/
*.swp
venv/
*.pyc
@@ -19,4 +19,6 @@ blogs/
.env
papers_already_written_on.txt
lib/papers_to_write_on
-workspace/
+workspace/web_research_reports/
+workspace/logs/
+workspace/personal/
diff --git a/README.md b/README.md
index bf77867e..d37e3353 100644
--- a/README.md
+++ b/README.md
@@ -6,9 +6,19 @@ This toolkit automates and enhances the process of blog creation, optimization,
## Getting Started
To use this tool, follow one of the Options below:
+---
+
+### Option 1: Cloud install: (I just want to write blogs..)
+
+Step 1). Make efforts to fork this present repo into your own account.
+
+Step 2). Follow this guide:
+https://docs.replit.com/programming-ide/using-git-on-replit/running-github-repositories-replit
+
---
-### Option 1: Local laptop Install: (I know what I am doing..)
+
+### Option 2: Local laptop Install: (I know what I am doing..)
Step 1. Clone this repository to your local machine.
@@ -24,14 +34,6 @@ Step 4. Once the tools is running it will guide/ask for your APIs. It will provi
---
-### Option 2: Cloud install: (I just want to write blogs..)
-
-Step 1). Make efforts to fork this present repo into your own account.
-
-Step 2). Follow this guide:
-https://docs.replit.com/programming-ide/using-git-on-replit/running-github-repositories-replit
-
-
### Option 3: Web URL: (Clickty Clickty Website, Free..)
Step 1). Error 404: Page not found.
diff --git a/alwrity.py b/alwrity.py
index 1c63f001..9c5f0afc 100644
--- a/alwrity.py
+++ b/alwrity.py
@@ -1,13 +1,18 @@
import os
from pathlib import Path
-import requests
import typer
-from PyInquirer import prompt
+from prompt_toolkit.shortcuts import checkboxlist_dialog, message_dialog, input_dialog
+from prompt_toolkit import prompt
+from prompt_toolkit.styles import Style
+from prompt_toolkit.shortcuts import radiolist_dialog
+from prompt_toolkit.clipboard.pyperclip import PyperclipClipboard
+from dotenv import load_dotenv
+import requests
from rich import print
+from rich.console import Console
from rich.text import Text
-from dotenv import load_dotenv
load_dotenv(Path('.env'))
app = typer.Typer()
@@ -19,105 +24,71 @@ from lib.ai_writers.keywords_to_blog import write_blog_from_keywords
def prompt_for_time_range():
os.system("clear" if os.name == "posix" else "cls")
- print("\nπ If you researching keywords that are recent than use accordingly, Default is Anytime.\n")
- questions = [
- {
- 'type': 'list',
- 'name': 'time_range',
- 'message': 'π Select Search result time range:',
- 'choices': ["anytime", "past year", "past month", "past week", "past day"],
- 'default': 'anytime'
- }
- ]
- answers = prompt(questions)
- return answers['time_range']
+ print("\nπ If you're researching keywords that are recent, use accordingly. Default is Anytime.\n")
+ choices = [("anytime", "Anytime"), ("past year", "Past Year"), ("past month", "Past Month"),
+ ("past week", "Past Week"), ("past day", "Past Day")]
+ selected_time_range = radiolist_dialog(title="Select Search result time range:", values=choices).run()
+ return selected_time_range[0] if selected_time_range else None
+
def write_blog_options():
- questions = [
- {
- 'type': 'list',
- 'name': 'blog_type',
- 'message': 'π Choose a blog type:',
- 'choices': ['Keywords', 'Audio YouTube', 'Programming',
- 'Scholar', 'News/TBD','Finance/TBD', 'Quit'],
- }
+ choices = [
+ ("Keywords", "Keywords"),
+ ("Audio YouTube", "Audio YouTube"),
+ ("Programming", "Programming"),
+ ("Scholar", "Scholar"),
+ ("News/TBD", "News/TBD"),
+ ("Finance/TBD", "Finance/TBD"),
+ ("Quit", "Quit")
]
- answers = prompt(questions)
- return answers['blog_type']
+ selected_blog_type = radiolist_dialog(title="Choose a blog type:", values=choices).run()
+ return selected_blog_type if selected_blog_type else None
@app.command()
def start_interactive_mode():
- """
- This function is executed when no command is provided.
- It prompts the user to choose between "Write Blog" and "Do Web Research."
- """
os.system("clear" if os.name == "posix" else "cls")
- text = Text()
- text.append("_______________________________________________________________________")
- text.append("\nβ οΈ Alert! π₯βπ₯\n", style="bold red")
- text.append("If you know what to write, choose 'Write Blog'\n", style="bold blue")
- text.append("If unsure, lets 'do web research' to write on\n", style="bold red")
- text.append("If Testing-it-out/getting-started, choose 'Blog Tools\n", style="bold green")
- text.append("_______________________________________________________________________\n")
-
+ text = "_______________________________________________________________________\n"
+ text += "\nβ οΈ Alert! π₯βπ₯\n"
+ text += "If you know what to write, choose 'Write Blog'\n"
+ text += "If unsure, let's 'do web research' to write on\n"
+ text += "If Testing-it-out/getting-started, choose 'Blog Tools\n"
+ text += "_______________________________________________________________________\n"
print(text)
- questions = [
- {
- 'type': 'list',
- 'name': 'mode',
- 'message': 'Choose an option:',
- 'choices': ['Write Blog', 'Do keyword Research', 'Create Blog Images',
- 'Competitor Analysis', 'Blog Tools', 'Social Media', 'Quit'],
- }
- ]
- answers = prompt(questions)
- mode = answers['mode']
- if mode == 'Write Blog':
- write_blog()
- elif mode == 'Do keyword Research':
- do_web_research()
- elif mode == 'Create Blog Images':
- faq_generator()
- elif mode == 'Competitor Analysis':
- # Metaphor similar search
- competitor_analysis()
- elif mode == 'Recent News Summarizer':
- print("""TBD: 1. Get tavily News.
- 2. Get metaphor news.
- 3. Get from NewsApi
- 4. Get YOU.com News.""")
- recent_news_summarizer()
- elif mode == 'Blog Tools':
- blog_tools()
- elif mode == 'Social Media':
- print("""
- #whatsapp
- #instagram
- #youtube
- #twitter/X
- #Linked-in posts
- """)
- raise typer.Exit()
- elif mode == 'Quit':
- typer.echo("Exiting, F*** Off!")
- raise typer.Exit()
-
-
-def get_api_key(api_key: str, api_description: str):
- """
- Ask the user to input the missing API key and add it to the .env file.
-
- Args:
- api_key (str): The name of the API key variable.
- api_description (str): The description of the API key.
- """
- user_input = typer.prompt(f"{api_description} is missing. Please enter {api_key} API Key:")
- with open(".env", "a") as env_file:
- env_file.write(f"{api_key}={user_input}\n")
- print(f"β
{api_description} API Key added to .env file.")
-
+ choices = [
+ ("Write Blog", "Write Blog"),
+ ("Do keyword Research", "Do keyword Research"),
+ ("Create Blog Images", "Create Blog Images"),
+ ("Competitor Analysis", "Competitor Analysis"),
+ ("Blog Tools", "Blog Tools"),
+ ("Social Media", "Social Media"),
+ ("Quit", "Quit")
+ ]
+ mode = radiolist_dialog(title="Choose an option:", values=choices).run()
+ if mode:
+ if mode == 'Write Blog':
+ write_blog()
+ elif mode == 'Do keyword Research':
+ do_web_research()
+ elif mode == 'Create Blog Images':
+ faq_generator()
+ elif mode == 'Competitor Analysis':
+ competitor_analysis()
+ elif mode == 'Blog Tools':
+ blog_tools()
+ elif mode == 'Social Media':
+ print("""
+ #whatsapp
+ #instagram
+ #youtube
+ #twitter/X
+ #Linked-in posts
+ """)
+ raise typer.Exit()
+ elif mode == 'Quit':
+ typer.echo("Exiting, Getting Lost!")
+ raise typer.Exit()
def check_search_apis():
@@ -125,6 +96,10 @@ def check_search_apis():
Check if necessary environment variables are present.
Display messages with links on how to get them if not present.
"""
+ # Create a Rich console
+ console = Console()
+
+ # Use rich.print for styling and hyperlinking
print("\n\nπβοΈ πβοΈ Before doing web research, ensure the following API keys are available:")
print("Blogen uses Basic, Semantic, Neural web search using above APIs for contextual blog generation.\n")
@@ -139,46 +114,33 @@ def check_search_apis():
with typer.progressbar(api_keys.items(), label="Checking API keys", length=len(api_keys)) as progress:
for key, description in progress:
if os.getenv(key) is None:
- print(f"[bold red]β π« {key} is missing:[/bold red] [link={key}]Get {key} API Key[/link]")
+ # Use rich.print for styling and hyperlinking
+ print(f"[bold red]β π« {key} is missing:[/bold red] [blue underline]Get {key} API Key[/blue underline]")
+ typer.echo(f"[bold red]β π« {key} is missing:[/bold red] [link={key}]Get {key} API Key[/link]")
missing_keys.append((key, description))
if missing_keys:
print("\nMost are Free APIs and really worth your while signing up for them.")
- print(":pile_of_poo: :pile_of_poo: GO GET THEM, on above urls. [bold red]")
- print("Note: They offer free/limited api calls, so we use most of them to have a lot of free api calls.")
- print("\n[bold red]TBD: Provide option to use user defined search engines.\n")
+ print("π©π©π©: GO GET THEM, on above urls. [bold red]")
+ #print("Note: They offer free/limited api calls, so we use most of them to have a lot of free api calls.")
for key, description in missing_keys:
get_api_key(key, description)
else:
return True
-def check_llm_environs():
- """ Function to check which LLM api is given. """
- gpt_provider = os.getenv("GPT_PROVIDER")
-
- if gpt_provider == "google":
- api_key_var = "GEMINI_API_KEY"
- missing_api_msg = f"To get your {api_key_var}, please visit: https://aistudio.google.com/app/apikey"
- elif gpt_provider == "openai":
- api_key_var = "OPENAI_API_KEY"
- missing_api_msg = "To get your OpenAI API key, please visit: https://openai.com/blog/openai-api"
- else:
- typer.echo("Unsupported GPT provider specified in GPT_PROVIDER environment variable.")
- return
+def get_api_key(api_key: str, api_description: str):
+ """
+ Ask the user to input the missing API key and add it to the .env file.
- if os.getenv(api_key_var) is None:
- typer.echo(f"The {api_key_var} environment variable is missing.")
- typer.echo(missing_api_msg)
- api_key = typer.prompt(f"Please enter your {api_key_var} API Key:")
- # Update .env file
- with open(".env", "a") as env_file:
- env_file.write(f"{api_key_var}={api_key}\n")
- typer.echo(f"{api_key_var} API Key added to .env file.")
- return
-
- if gpt_provider == "openai" and os.getenv("OPENAI_API_KEY") is None:
- typer.echo("To get your OpenAI API key, please visit: https://openai.com/blog/openai-api")
+ Args:
+ api_key (str): The name of the API key variable.
+ api_description (str): The description of the API key.
+ """
+ user_input = typer.prompt(f"\nππPlease enter {api_key} API Key:")
+ with open(".env", "a") as env_file:
+ env_file.write(f"{api_key}={user_input}\n")
+ print(f"β
{api_description} API Key added to .env file.")
def faq_generator():
@@ -186,46 +148,45 @@ def faq_generator():
def blog_tools():
- """ Blogging Aid Tools """
os.system("clear" if os.name == "posix" else "cls")
- text = Text()
- text.append("_______________________________________________________________________")
- text.append("\nβ οΈ Alert! π₯βπ₯\n", style="bold red")
- text.append("Collection of Helpful Blogging Tools, powered by LLMs.\n", style="bold green")
- text.append("_______________________________________________________________________\n")
-
+ text = "_______________________________________________________________________\n"
+ text += "\nβ οΈ Alert! π₯βπ₯\n"
+ text += "Collection of Helpful Blogging Tools, powered by LLMs.\n"
+ text += "_______________________________________________________________________\n"
print(text)
- # https://developers.google.com/speed/docs/insights/v5/get-started
- questions = [
- {
- 'type': 'list',
- 'name': 'mode',
- 'message': 'Choose a Blogging Tool:',
- 'choices': ['Write Blog Title', 'Write Blog Meta Description', 'Write Blog Introduction',
- 'Write Blog conclusion', 'Write Blog Outline', 'Generate Blog FAQs', 'Research blog referances',
- 'Convert Blog To HTML', 'Convert Blog To Markdown', 'Blog Proof Reader',
- 'Get Blog Tags', 'Get blog categories', 'Get Blog Code Examples', 'Check WebPage Performance',
- 'Quit/Exit',],
- }
+ choices = [
+ ("Write Blog Title", "Write Blog Title"),
+ ("Write Blog Meta Description", "Write Blog Meta Description"),
+ ("Write Blog Introduction", "Write Blog Introduction"),
+ ("Write Blog conclusion", "Write Blog conclusion"),
+ ("Write Blog Outline", "Write Blog Outline"),
+ ("Generate Blog FAQs", "Generate Blog FAQs"),
+ ("Research blog references", "Research blog references"),
+ ("Convert Blog To HTML", "Convert Blog To HTML"),
+ ("Convert Blog To Markdown", "Convert Blog To Markdown"),
+ ("Blog Proof Reader", "Blog Proof Reader"),
+ ("Get Blog Tags", "Get Blog Tags"),
+ ("Get blog categories", "Get blog categories"),
+ ("Get Blog Code Examples", "Get Blog Code Examples"),
+ ("Check WebPage Performance", "Check WebPage Performance"),
+ ("Quit/Exit", "Quit/Exit")
]
- answers = prompt(questions)
- mode = answers['mode']
- if mode == 'Write Blog Title':
- return
+ selected_tool = radiolist_dialog(title="Choose a Blogging Tool:", values=choices).run()
+ if selected_tool:
+ tool = selected_tool[0]
+ if tool == 'Write Blog Title':
+ return
+
-
def competitor_analysis():
- """ Do metaphor similar search """
- text = Text()
- text.append("_______________________________________________________________________")
- text.append("\nβ οΈ Alert! π₯βπ₯\n", style="bold red")
- text.append("Provide competitor's URL, get details of similar/alternative companies.\n", style="bold red")
- text.append("Usecases: Know similar companies and alternatives, to given URL\n", style="bold blue")
- text.append("_______________________________________________________________________\n")
+ text = "_______________________________________________________________________\n"
+ text += "\nβ οΈ Alert! π₯βπ₯\n"
+ text += "Provide competitor's URL, get details of similar/alternative companies.\n"
+ text += "Usecases: Know similar companies and alternatives, to given URL\n"
+ text += "_______________________________________________________________________\n"
print(text)
- similar_url = typer.prompt(f"Enter Valid URL to get web analysis")
-
+ similar_url = prompt("Enter Valid URL to get web analysis")
try:
metaphor_find_similar(similar_url)
except Exception as err:
@@ -234,96 +195,148 @@ def competitor_analysis():
def write_blog():
- """
- Write Blog option with sub-options like Keywords, Audio YouTube, GitHub, and Scholar.
- """
blog_type = write_blog_options()
-
- if blog_type == 'Keywords':
- blog_from_keyword()
- elif blog_type == 'Audio YouTube':
- audio_youtube = typer.prompt("Enter YouTube URL for audio blog generation:")
- print(f"Write audio blog based on YouTube URL: {audio_youtube}")
- elif blog_type == 'GitHub':
- github = typer.prompt("Enter GitHub URL, CSV file, or topic:")
- print(f"Write blog based on GitHub: {github}")
- elif blog_type == 'Scholar':
- scholar = typer.prompt("Enter research papers keywords:")
- print(f"Write blog based on scholar: {scholar}")
- elif blog_type == 'Quit':
- typer.echo("Exiting, F*** Off!")
- raise typer.Exit()
+ if blog_type:
+ if blog_type == 'Keywords':
+ blog_from_keyword()
+ elif blog_type == 'Audio YouTube':
+ audio_youtube = prompt("Enter YouTube URL for audio blog generation:")
+ print(f"Write audio blog based on YouTube URL: {audio_youtube}")
+ elif blog_type == 'GitHub':
+ github = prompt("Enter GitHub URL, CSV file, or topic:")
+ print(f"Write blog based on GitHub: {github}")
+ elif blog_type == 'Scholar':
+ scholar = prompt("Enter research papers keywords:")
+ print(f"Write blog based on scholar: {scholar}")
+ elif blog_type == 'Quit':
+ typer.echo("Exiting, Getting Lost..")
+ raise typer.Exit()
def blog_from_keyword():
- """ Write blog from given keyword. """
- print("Write blog based on keywords.")
- check_llm_environs()
- keywords = typer.prompt("Enter 'keywords/Blog Title' for blog generation:")
- final_blog = write_blog_from_keywords(keywords)
+ """ Input blog keywords, research and write a factual blog."""
+ while True:
+ print("________________________________________________________________")
+ blog_keywords = input_dialog(
+ title='Enter Keywords/Blog Title',
+ text='Shit in, Shit Out; Better keywords, better research, hence better content.\nπ Enter keywords/Blog Title for blog generation:',
+ ).run()
+
+ # If the user cancels, exit the loop
+ if blog_keywords is None:
+ break
+ if blog_keywords and len(blog_keywords.split()) >= 2:
+ break
+ else:
+ message_dialog(
+ title='Warning',
+ text='π« Blog keywords should be at least two words long. Please try again.'
+ ).run()
+ if blog_keywords:
+ try:
+ write_blog_from_keywords(blog_keywords)
+ except Exception as err:
+ print(f"Failed to write blog on {blog_keywords}, Error: {err}\n")
+ exit(1)
def do_web_research():
- """
- Do Web Research option with time_range, search_keywords, and include_urls sub-options.
- """
+ """ Input keywords and do web research and present a report."""
if check_search_apis():
while True:
print("________________________________________________________________")
- search_keywords = typer.prompt("π Enter keywords for web research:")
- # Giving a single keywords, yields bad results.
+ search_keywords = input_dialog(
+ title='Enter Search Keywords below:',
+ text='π Enter keywords for web research (Or keywords from your blog):',
+ ).run()
if search_keywords and len(search_keywords.split()) >= 2:
break
else:
- print("π« Search keywords should be at least three words long. Please try again.")
+ message_dialog(
+ title='Warning',
+ text='π« Search keywords should be at least three words long. Please try again.'
+ ).run()
+ selected_time_range = prompt_for_time_range()
- # Display available choices
-# print("Choose from the following options:")
-# search_keyword_choices = ["choice1", "choice2", "choice3"]
-# for i, choice in enumerate(search_keyword_choices, start=1):
-# print(f"{i}. '{choice}'")
-#
-# choice_index = typer.prompt("Enter the NUMBER to choose which keywords to use:")
-#
-# try:
-# choice_index = int(choice_index)
-# if 1 <= choice_index <= len(search_keyword_choices):
-# search_keywords = search_keyword_choices[choice_index - 1]
-# break
-# else:
-# print("π« Invalid choice. Please try again.")
-# except ValueError:
-# print("π« Invalid input. Please enter a valid number.")
+ # Display input dialog for similar search URL (optional)
+ similar_url = input_dialog(
+ title="Enter a similar search URL",
+ text="π Enter a similar search URL (Optional: Enter to skip):\nπUsecases: Competitor Analysis Tool. π‘Discover similar companies, startups and technologies.",
+ default="",
+ ).run()
-
- print("________________________________________________________________")
- time_range = prompt_for_time_range()
+ # Display input dialog for included URLs (optional)
+ include_urls = input_dialog(
+ title="Enter URLs to include in the web search:",
+ text="π Enter comma-separated URLs to include in web research (press Enter to skip):\nπ If you wish to [bold]confine search[/bold] to certain domains like wikipedia etc.",
+ default="",
+ ).run()
- os.system("clear" if os.name == "posix" else "cls")
- print("\n________________________________________________________________")
- print("\nπ Include a [green]URL[/green] to get [bold]similar/semantic[/bold]. For example, competitor's url.")
- print("π‘ Usecases: Competitor Analysis Tool. Discover similar companies, startups and technologies.\n")
- similar_url = typer.prompt("π Enter a similar search URL (press Enter to continue):", default="")
- os.system("clear" if os.name == "posix" else "cls")
- print("\n________________________________________________________________")
- print("\nπ If you wish to [bold]confine search[/bold] to certain domains like wikipedia etc.\n")
- include_urls = typer.prompt("π Enter comma-separated URLs to include in web research (press Enter if none):", default="")
-
- try:
- print(f"ππ [bold green]Starting web research on given keywords: {search_keywords}..")
- #print(f"Web Research: Time Range - {time_range}, Search Keywords - {search_keywords}, Include URLs - {include_urls}")
- web_research_result = gpt_web_researcher(search_keywords,
- time_range=time_range,
- include_domains=include_urls,
- similar_url=similar_url)
- except Exception as err:
- print(f"\nπ₯π€― [bold red]ERROR π€― : Failed to do web research: {err}\n")
+ try:
+ print(f"ππ¬π [bold green]Starting web research on given keywords: {search_keywords}..")
+ #print(f"Web Research: Time Range - {time_range}, Search Keywords - {search_keywords}, Include URLs - {include_urls}")
+ web_research_result = gpt_web_researcher(search_keywords,
+ time_range=selected_time_range,
+ include_domains=include_urls,
+ similar_url=similar_url)
+ except Exception as err:
+ print(f"\nπ₯π€― [bold red]ERROR π€― : Failed to do web research: {err}\n")
+
+
+def check_llm_environs():
+ """ Function to check which LLM api is given. """
+ # Check if GPT_PROVIDER is defined in .env file
+ gpt_provider = os.getenv("GPT_PROVIDER")
+
+ # Load .env file
+ load_dotenv()
+
+ # Disable unsupported GPT providers
+ supported_providers = ['google', 'openai', 'mistralai']
+ if gpt_provider is None or gpt_provider.lower() not in supported_providers:
+ #message_dialog(
+ # title="Unsupported GPT Provider",
+ # text="GPT_PROVIDER is not set or has an unsupported value."
+ #).run()
+
+ # Prompt user to select a provider
+ selected_provider = radiolist_dialog(
+ title='Select your preferred GPT provider:',
+ text="Please choose GPT provider Below:\nπΊGoogle Gemini recommended, its π.",
+ values=[
+ ("Google", "google"),
+ ("Openai", "openai"),
+ ("MistralAI/WIP", "mistralai/WIP"),
+ ("Ollama", "Ollama (TBD)")
+ ]
+ ).run()
+ if selected_provider:
+ gpt_provider = selected_provider
+
+ if gpt_provider.lower() == "google":
+ api_key_var = "GEMINI_API_KEY"
+ missing_api_msg = f"To get your {api_key_var}, please visit: https://aistudio.google.com/app/apikey"
+ elif gpt_provider.lower() == "openai":
+ api_key_var = "OPENAI_API_KEY"
+ missing_api_msg = "To get your OpenAI API key, please visit: https://openai.com/blog/openai-api"
+ elif gpt_provider.lower() == "mistralai":
+ api_key_var = "MISTRAL_API_KEY"
+ missing_api_msg = "To get your MistralAI API key, please visit: https://mistralai.com/api"
+
+ if os.getenv(api_key_var) is None:
+ # Ask for the API key
+ print(f"π«The {api_key_var} is missing. {missing_api_msg}")
+ api_key = typer.prompt(f"\nππPlease enter {api_key_var} API Key:")
+
+ # Update .env file
+ with open(".env", "a") as env_file:
+ env_file.write(f"GPT_PROVIDER={gpt_provider.lower()}\n")
+ env_file.write(f"{api_key_var}={api_key}\n")
def check_internet():
try:
- # Attempt to send a GET request to a well-known website
response = requests.get("http://www.google.com", timeout=20)
if not response.status_code == 200:
print("π₯π€― WTFish, Internet is NOT available. Enjoy the wilderness..")
@@ -340,8 +353,25 @@ def check_internet():
print("Internet: An error occurred:", e)
exit(1)
+
+def create_env_file():
+ env_file = Path('.env')
+ if not env_file.is_file():
+ try:
+ with open('.env', 'w') as f:
+ f.write('# Alwrity will add your environment variables here\n')
+ except Exception as e:
+ print(f"π₯π€―Error occurred while creating .env file: {e}")
+
+
if __name__ == "__main__":
+ print("Checking Internet, lets get the basics right.")
check_internet()
+ print("Create .env file, if not Present working directory")
+ create_env_file()
+ print("Check Metaphor, Tavily, YOU.com Search API keys.")
check_search_apis()
+ print("Check LLM details & AI Model to use.")
check_llm_environs()
+ load_dotenv(Path('.env'))
app()
diff --git a/lib/ai_web_researcher/google_serp_search.py b/lib/ai_web_researcher/google_serp_search.py
index bcd6228a..fa061f5e 100644
--- a/lib/ai_web_researcher/google_serp_search.py
+++ b/lib/ai_web_researcher/google_serp_search.py
@@ -66,23 +66,23 @@ def google_search(query):
Returns:
list: List of search results based on the specified flag.
"""
- try:
- perform_serpapi_google_search(query)
- logger.info(f"FIXME: Google serapi: {query}")
- #return process_search_results(search_result)
- except Exception as err:
- logger.error(f"ERROR: Check Here: https://serpapi.com/. Your requests may be over. {err}")
+ #try:
+ # perform_serpapi_google_search(query)
+ # logger.info(f"FIXME: Google serapi: {query}")
+ # #return process_search_results(search_result)
+ #except Exception as err:
+ # logger.error(f"ERROR: Check Here: https://serpapi.com/. Your requests may be over. {err}")
# Retry with serper.dev
try:
logger.info("Trying Google search with Serper.dev: https://serper.dev/api-key")
search_result = perform_serperdev_google_search(query)
process_search_results(search_result)
+ return(search_result)
except Exception as err:
logger.error(f"Failed to do Google search with serper.dev: {err}")
- return(search_result)
-
+
# # Retry with BROWSERLESS API
# try:
# search_result = perform_browserless_google_search(query)
@@ -118,7 +118,10 @@ def perform_serpapi_google_search(query, location="in"):
try:
# Check if API key is provided
if not os.getenv("SERPAPI_KEY"):
- raise ValueError("SERPAPI_KEY key is required for SerpApi")
+ #raise ValueError("SERPAPI_KEY key is required for SerpApi")
+ logger.error("SERPAPI_KEY key is required for SerpApi")
+ return
+
# Create a GoogleSearch instance
search = GoogleSearch({
@@ -164,7 +167,7 @@ def perform_serperdev_google_search(query):
"q": query,
"gl": "in",
"hl": "en",
- "num": 5,
+ "num": 10,
"autocorrect": True,
"page": 1,
"type": "search",
diff --git a/lib/ai_web_researcher/google_trends_researcher.py b/lib/ai_web_researcher/google_trends_researcher.py
index 29588b3f..d0e4d832 100644
--- a/lib/ai_web_researcher/google_trends_researcher.py
+++ b/lib/ai_web_researcher/google_trends_researcher.py
@@ -23,6 +23,8 @@ Note: Ensure that the required libraries are installed using 'pip install pytren
"""
import os
+import time # I wish
+import random
import requests
import numpy as np
import sys
@@ -186,6 +188,7 @@ def get_related_topics_and_save_csv(search_keywords):
data = pytrends.related_topics()
except Exception as err:
logger.error(f"Failed to get pytrends realted topics: {err}")
+ return
# Extract data from the result
top_topics = list(data.values())[0]['top']
rising_topics = list(data.values())[0]['rising']
@@ -238,6 +241,8 @@ def get_results(query):
try:
query = urllib.parse.quote_plus(query)
response = get_source(f"https://suggestqueries.google.com/complete/search?output=chrome&hl=en&q={query}")
+ time.sleep(random.uniform(0.1, 0.6))
+
if response:
response.raise_for_status()
results = json.loads(response.text)
@@ -501,6 +506,8 @@ def do_google_trends_analysis(search_term):
else:
all_the_keywords.append(suggestions_df['Keywords'].tolist())
all_the_keywords = ','.join([', '.join(filter(None, map(str, sublist))) for sublist in all_the_keywords])
+ # Generate a random sleep time between 2 and 3 seconds
+ time.sleep(random.uniform(2, 3))
#
# # FIXME: Get result from vision GPT. Fetch and visualize Google Trends data
@@ -510,12 +517,16 @@ def do_google_trends_analysis(search_term):
# result_df = plot_interest_by_region(search_term)
#
# Display additional information
- result_df = get_related_topics_and_save_csv(search_term)
- # Extract 'Top' topic_title
- top_topic_title = result_df['topic_title'].values.tolist()
- # Join each sublist into one string separated by comma
- #top_topic_title = [','.join(filter(None, map(str, sublist))) for sublist in top_topic_title]
- top_topic_title = ','.join([', '.join(filter(None, map(str, sublist))) for sublist in top_topic_title])
+ try:
+ result_df = get_related_topics_and_save_csv(search_term)
+ # Extract 'Top' topic_title
+ if result_df:
+ top_topic_title = result_df['topic_title'].values.tolist()
+ # Join each sublist into one string separated by comma
+ #top_topic_title = [','.join(filter(None, map(str, sublist))) for sublist in top_topic_title]
+ top_topic_title = ','.join([', '.join(filter(None, map(str, sublist))) for sublist in top_topic_title])
+ except Exception as err:
+ logger.error(f"Failed to get results from google trends related topics: {err}")
# TBD: Not getting great results OR unable to understand them.
#all_the_keywords += top_topic_title
diff --git a/lib/ai_web_researcher/gpt_online_researcher.py b/lib/ai_web_researcher/gpt_online_researcher.py
index 2d8bd884..25f0f427 100644
--- a/lib/ai_web_researcher/gpt_online_researcher.py
+++ b/lib/ai_web_researcher/gpt_online_researcher.py
@@ -9,7 +9,6 @@ import json
from pathlib import Path
import sys
from typing import List, NamedTuple
-from loguru import logger
from datetime import datetime
from ..gpt_providers.gemini_pro_text import gemini_text_response
@@ -17,8 +16,9 @@ from .tavily_ai_search import get_tavilyai_results
from .metaphor_basic_neural_web_search import metaphor_find_similar, metaphor_search_articles
from .google_serp_search import google_search
from .google_trends_researcher import do_google_trends_analysis
-from .web_research_report import write_web_research_report
+#from .web_research_report import write_web_research_report
+from loguru import logger
# Configure logger
logger.remove()
logger.add(sys.stdout,
diff --git a/lib/ai_web_researcher/tavily_ai_search.py b/lib/ai_web_researcher/tavily_ai_search.py
index 401b6365..b4a86995 100644
--- a/lib/ai_web_researcher/tavily_ai_search.py
+++ b/lib/ai_web_researcher/tavily_ai_search.py
@@ -66,7 +66,7 @@ def get_tavilyai_results(keywords, include_urls, search_depth="advanced"):
# Retrieve API keys
api_key = os.getenv('TAVILY_API_KEY')
if not api_key:
- raise ValueError("API keys for Tavily or OpenAI are not set.")
+ raise ValueError("API keys for Tavily is Not set.")
# Initialize Tavily client
try:
diff --git a/lib/ai_web_researcher/web_research_report.py b/lib/ai_web_researcher/web_research_report.py
index 09944523..b59b2d85 100644
--- a/lib/ai_web_researcher/web_research_report.py
+++ b/lib/ai_web_researcher/web_research_report.py
@@ -1,11 +1,10 @@
-from langchain.adapters.openai import convert_openai_messages
-from langchain.chat_models import ChatOpenAI
-
+import os
from ..gpt_providers.gemini_pro_text import gemini_text_response
-def write_web_research_report(web_research, faq_questions, gpt_provider="gemini"):
+def write_web_research_report(web_research, faq_questions):
""" """
+ gpt_provider = os.environ["GPT_PROVIDER"]
if "gemini" in gpt_provider:
prompt = ["You are an SEO and marketing expert, who writes unique, factual and comprehensive research reports."
"I will provide you web research report as json data and a list of related FAQ questions."
diff --git a/lib/ai_writers/blog_from_google_serp.py b/lib/ai_writers/blog_from_google_serp.py
index 8724814f..0bba303c 100644
--- a/lib/ai_writers/blog_from_google_serp.py
+++ b/lib/ai_writers/blog_from_google_serp.py
@@ -34,14 +34,14 @@ def write_blog_google_serp(search_keyword, search_results):
Google search Result: "{search_results}"
"""
logger.info("Generating blog and FAQs from web search result.")
- if 'google' in gpt_providers:
+ if 'google' in gpt_providers.lower():
try:
response = gemini_text_response(prompt)
return response
except Exception as err:
logger.error(f"Failed to get response from gemini: {err}")
raise err
- elif 'openai' in gpt_providers:
+ elif 'openai' in gpt_providers.lower():
try:
logger.info("Calling OpenAI LLM.")
response = openai_chatgpt(prompt)
diff --git a/lib/ai_writers/combine_blog_and_keywords.py b/lib/ai_writers/combine_blog_and_keywords.py
index 48c9694e..db25aa2b 100644
--- a/lib/ai_writers/combine_blog_and_keywords.py
+++ b/lib/ai_writers/combine_blog_and_keywords.py
@@ -30,7 +30,7 @@ def blog_with_keywords(blog, keywords):
list of keywords: '{keywords}'
"""
- if 'google' in gpt_providers:
+ if 'google' in gpt_providers.lower():
prompt = f"""You are an expert copywriter specializing in content optimization for SEO.
I will provide you with my 'blog content' and 'list of keywords' on the same topic.
Your task is to write an original blog, using the given keywords and blog content.
@@ -39,7 +39,6 @@ def blog_with_keywords(blog, keywords):
Always, include figures, data, results from given content.
It is important that your blog is original and unique. It should be highly readable and SEO optimized.
-
Blog content: '{blog}'
list of keywords: '{keywords}'
"""
@@ -49,7 +48,7 @@ def blog_with_keywords(blog, keywords):
except Exception as err:
logger.error(f"Failed to get response from gemini: {err}")
raise err
- elif 'openai' in gpt_providers:
+ elif 'openai' in gpt_providers.lower():
try:
logger.info("Calling OpenAI LLM.")
response = openai_chatgpt(prompt)
diff --git a/lib/ai_writers/combine_research_and_blog.py b/lib/ai_writers/combine_research_and_blog.py
index e390bb8a..3c0eefc6 100644
--- a/lib/ai_writers/combine_research_and_blog.py
+++ b/lib/ai_writers/combine_research_and_blog.py
@@ -20,10 +20,10 @@ def blog_with_research(report, blog):
"""Combine the given online research and gpt blog content"""
gpt_providers = os.environ["GPT_PROVIDER"]
prompt = f"""
- You are an expert copywriter specializing in content optimization for SEO.
+ You are an expert copywriter specializing in SEO content optimization for blogs.
I will provide you with a 'research report' and a 'blog content' on the same topic.
- Your task is to transform and combine the given research and blog content into a well-structured markdown, unique
- and engaging blog article.
+ Your task is to transform and combine the given 'research report' and 'blog content' into a well-structured, unique
+ and original blog article.
Your objectives include:
1. Master the report and blog content: Understand main ideas, key points, and the core message.
@@ -47,11 +47,11 @@ def blog_with_research(report, blog):
that will rank well in search engine results and engage readers effectively.
Create a blog post, in markdown, from the given research report and blog content below.
- Research report: {report}
- Blog content: {blog}
+ Research report: '{report}'
+ Blog content: '{blog}'
"""
- if 'google' in gpt_providers:
+ if 'google' in gpt_providers.lower():
prompt = f"""You are an expert copywriter specializing in content optimization for SEO.
I will provide you with my 'research report' and 'blog content' on the same topic.
Your task is to transform and combine the given research and blog content into a blog article.
@@ -70,7 +70,7 @@ def blog_with_research(report, blog):
except Exception as err:
logger.error(f"Failed to get response from gemini: {err}")
raise err
- elif 'openai' in gpt_providers:
+ elif 'openai' in gpt_providers.lower():
try:
logger.info("Calling OpenAI LLM.")
response = openai_chatgpt(prompt)
@@ -78,3 +78,6 @@ def blog_with_research(report, blog):
except Exception as err:
logger.error(f"failed to get response from Openai: {err}")
raise err
+ else:
+ logger.error(f"Unrecognised/Un-Supoorted GPT_PROVIDER: {gpt_providers}\n")
+ return
diff --git a/lib/ai_writers/gpt_blog_sections.py b/lib/ai_writers/gpt_blog_sections.py
index fbfb1a81..c0f93829 100644
--- a/lib/ai_writers/gpt_blog_sections.py
+++ b/lib/ai_writers/gpt_blog_sections.py
@@ -1,4 +1,5 @@
import sys
+import os
import json
from ..gpt_providers.openai_chat_completion import openai_chatgpt
@@ -13,9 +14,9 @@ logger.add(sys.stdout,
# FIXME: Provide num_blogs, num_faqs as inputs.
-def get_blog_sections_from_websearch(search_keyword, search_results, gpt_providers="gemini"):
+def get_blog_sections_from_websearch(search_keyword, search_results):
"""Combine the given online research and gpt blog content"""
-
+ gpt_providers = os.environ["GPT_PROVIDER"]
prompt = f"""
As a SEO expert and content writer, I will provide you with a search keyword and its google search result.
Your task is to write a blog title and 5 blog sub titles, from the given google search result.
diff --git a/lib/ai_writers/keywords_to_blog.py b/lib/ai_writers/keywords_to_blog.py
index 005d7cd4..3221e073 100644
--- a/lib/ai_writers/keywords_to_blog.py
+++ b/lib/ai_writers/keywords_to_blog.py
@@ -1,5 +1,6 @@
import sys
import os
+from textwrap import dedent
from pathlib import Path
from datetime import datetime
@@ -32,41 +33,42 @@ def write_blog_from_keywords(search_keywords, url=None):
# TBD: Keeping the results directory as fixed, for now.
os.environ["SEARCH_SAVE_FILE"] = os.path.join(os.getcwd(), "workspace", "web_research_reports",
search_keywords.replace(" ", "_") + "_" + datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
- logger.info(f"Researching and Writing Blog on keywords: {search_keywords}")
# Use to store the blog in a string, to save in a *.md file.
blog_markdown_str = ""
example_blog_titles = []
+
+ logger.info(f"Researching and Writing Blog on keywords: {search_keywords}")
# Call on the got-researcher, tavily apis for this. Do google search for organic competition.
google_search_result, g_titles = do_google_serp_search(search_keywords)
example_blog_titles.append(g_titles)
blog_markdown_str = write_blog_google_serp(search_keywords, google_search_result)
# logger.info/check the final blog content.
- logger.info(f"Final blog content: {blog_markdown_str}")
+ logger.info(f"######### Blog content Google SERP research: ###########\n\n{blog_markdown_str}\n\n")
# Do Tavily AI research to augument the above blog.
tavily_search_result, t_titles = do_tavily_ai_search(search_keywords)
example_blog_titles.append(t_titles)
- blog_markdown_str = blog_with_research(blog_markdown_str, tavily_search_result)
- logger.info(f"Final blog content: {blog_markdown_str}")
+ if tavily_search_result:
+ blog_markdown_str = blog_with_research(blog_markdown_str, tavily_search_result)
+ logger.info(f"######### Blog content after Tavily AI research: ######### \n\n{blog_markdown_str}\n\n")
try:
# Do Metaphor/Exa AI search.
metaphor_search_result, m_titles = do_metaphor_ai_research(search_keywords)
example_blog_titles.append(m_titles)
blog_markdown_str = blog_with_research(blog_markdown_str, metaphor_search_result)
- logger.info(f"Final blog content: {blog_markdown_str}")
+ logger.info(f"######## Blog content after EXA AI research: ########## \n\n{blog_markdown_str}\n\n")
except Exception as err:
logger.error(f"Failed to do Metaphor AI search: {err}")
# Do Google trends analysis and combine with latest blog.
try:
pytrends_search_result = do_google_pytrends_analysis(search_keywords)
+ logger.info(f"Google Trends keywords to use in the blog: {pytrends_search_result}\n")
blog_markdown_str = blog_with_keywords(blog_markdown_str, pytrends_search_result)
except Exception as err:
logger.error(f"Failed to do Google Trends Analysis:{err}")
-
- blog_markdown_str = blog_proof_editor(blog_markdown_str, search_keywords)
- logger.info(f"Final blog content: {blog_markdown_str}")
+ logger.info(f"########### Blog Content After Google Trends Analysis:######### \n {blog_markdown_str}\n\n")
# Combine YOU.com RAG search with the latest blog content.
#you_rag_result = get_rag_results(search_keywords)
@@ -74,6 +76,8 @@ def write_blog_from_keywords(search_keywords, url=None):
#blog_markdown_str = blog_with_research(blog_markdown_str, you_search_result)
#logger.info(f"Final blog content: {blog_markdown_str}")
+ blog_markdown_str = blog_proof_editor(blog_markdown_str, search_keywords)
+
blog_title, blog_meta_desc, blog_tags, blog_categories = blog_metadata(blog_markdown_str,
search_keywords, example_blog_titles)
@@ -92,4 +96,12 @@ def write_blog_from_keywords(search_keywords, url=None):
# TBD: Save the blog content as a .md file. Markdown or HTML ?
save_blog_to_file(blog_markdown_str, blog_title, blog_meta_desc, blog_tags, blog_categories, generated_image_filepath)
+ blog_frontmatter = dedent(f"""\n\n\n\
+ ---
+ title: {blog_title}
+ categories: [{blog_categories}]
+ tags: [{blog_tags}]
+ Meta description: {blog_meta_desc.replace(":", "-")}
+ ---\n\n""")
+ logger.info(f"{blog_frontmatter}{blog_markdown_str}")
logger.info(f"\n\n ################ Finished writing Blog for : {search_keywords} #################### \n")
diff --git a/lib/blog_metadata/get_blog_category.py b/lib/blog_metadata/get_blog_category.py
index d1e2c959..52a1b50a 100644
--- a/lib/blog_metadata/get_blog_category.py
+++ b/lib/blog_metadata/get_blog_category.py
@@ -27,13 +27,13 @@ def get_blog_categories(blog_article):
The blog content is: '{blog_article}'"
"""
logger.info("Generating blog categories for the given blog.")
- if 'google' in gpt_providers:
+ if 'google' in gpt_providers.lower():
try:
response = gemini_text_response(prompt)
return response
except Exception as err:
logger.error(f"Failed to get response from gemini: {err}")
- elif 'openai' in gpt_providers:
+ elif 'openai' in gpt_providers.lower():
try:
response = openai_chatgpt(prompt)
return response
diff --git a/lib/blog_metadata/get_blog_meta_desc.py b/lib/blog_metadata/get_blog_meta_desc.py
index 68f2d586..ede8a09d 100644
--- a/lib/blog_metadata/get_blog_meta_desc.py
+++ b/lib/blog_metadata/get_blog_meta_desc.py
@@ -27,13 +27,13 @@ def generate_blog_description(blog_content):
Respond with only one of your best effort and do not include your explanations.
Blog Content: '{blog_content}'"""
- if 'google' in gpt_providers:
+ if 'google' in gpt_providers.lower():
try:
response = gemini_text_response(prompt)
return response
except Exception as err:
logger.error("Failed to get response from gemini.")
- elif 'openai' in gpt_providers:
+ elif 'openai' in gpt_providers.lower():
try:
response = openai_chatgpt(prompt)
return response
diff --git a/lib/blog_metadata/get_blog_title.py b/lib/blog_metadata/get_blog_title.py
index 15be0c1e..8c39189c 100644
--- a/lib/blog_metadata/get_blog_title.py
+++ b/lib/blog_metadata/get_blog_title.py
@@ -42,13 +42,22 @@ def generate_blog_title(blog_article, keywords=None, example_titles=None, num_ti
Blog Keywords: '{keywords}'
Example Titles: '{example_titles}'
"""
- if 'google' in gpt_providers:
+ elif not example_titles:
+ prompt = prompt = f"""As a SEO expert, I will provide you with my blog article.
+ Your task is to write {num_titles} blog title.
+ Follow SEO best practises to suggest the blog title.
+ Please keep the titles concise, not exceeding 60 words.
+ Respond with only {num_titles} title and no explanations.
+ Negative Keywords: Unvieling, unleash, power of. Dont use such words in your title.
+ Blog Article: '{keywords}'
+ """
+ if 'google' in gpt_providers.lower():
try:
response = gemini_text_response(prompt)
return response
except Exception as err:
logger.error(f"Failed to get response from gemini: {err}")
- elif 'openai' in gpt_providers:
+ elif 'openai' in gpt_providers.lower():
try:
logger.info("Calling OpenAI LLM.")
response = openai_chatgpt(prompt)
diff --git a/lib/blog_metadata/get_tags.py b/lib/blog_metadata/get_tags.py
index 58778718..69e49e4f 100644
--- a/lib/blog_metadata/get_tags.py
+++ b/lib/blog_metadata/get_tags.py
@@ -25,13 +25,13 @@ def get_blog_tags(blog_article):
for the given blog content. Only reply with comma separated values.
Blog content: {blog_article}."""
logger.info("Generating Blog tags for the given blog post.")
- if 'google' in gpt_providers:
+ if 'google' in gpt_providers.lower():
try:
response = gemini_text_response(prompt)
return response
except Exception as err:
logger.error("Failed to get response from gemini.")
- elif 'openai' in gpt_providers:
+ elif 'openai' in gpt_providers.lower():
try:
response = openai_chatgpt(prompt)
return response
diff --git a/lib/blog_postprocessing/blog_proof_reader.py b/lib/blog_postprocessing/blog_proof_reader.py
index 04e64dd7..b2a0d698 100644
--- a/lib/blog_postprocessing/blog_proof_reader.py
+++ b/lib/blog_postprocessing/blog_proof_reader.py
@@ -26,19 +26,19 @@ def blog_proof_editor(blog_content, blog_keywords):
4). Tone and Brand Alignment: Adjust the tone, voice, personality of given content to make it unique.
5). Optimize Content Structure: Reorganize the content for a more impactful presentation,
including better paragraphing and transitions.
- 6). Simplify given content: Simplify concepts and replace overly complex jargons and words.
+ 6). Simplify content: Simplify concepts and replace overly complex words. Use simple english words.
7). Refine Overall Structure: Make structural changes to improve the overall impact of the content.
\n\nMain keywords: '{blog_keywords}'
My Blog: '{blog_content}'. """
- if 'openai' in gpt_provider:
+ if 'openai' in gpt_provider.lower():
try:
response = openai_chatgpt(prompt)
return response
except Exception as err:
SystemError(f"Openai Error Blog Proof Reading: {err}")
- elif 'google' in gpt_provider:
+ elif 'google' in gpt_provider.lower():
try:
response = gemini_text_response(prompt)
return response
diff --git a/lib/blog_postprocessing/convert_content_to_markdown.py b/lib/blog_postprocessing/convert_content_to_markdown.py
index 2c1bfb73..f70c47db 100644
--- a/lib/blog_postprocessing/convert_content_to_markdown.py
+++ b/lib/blog_postprocessing/convert_content_to_markdown.py
@@ -57,13 +57,13 @@ def convert_tomarkdown_format(blog_content, gpt_provider="openai"):
Blog Post: '{blog_content}'"""
- if 'openai' in gpt_provider:
+ if 'openai' in gpt_provider.lower():
try:
response = openai_chatgpt(prompt)
return response
except Exception as err:
SystemError(f"Openai Error in converting to Markdown format.")
- elif 'gemini' in gpt_provider:
+ elif 'gemini' in gpt_provider.lower():
prompt = f""" Convert the given blog post into well structured MARKDOWN content.
Do not alter the given blog post.
diff --git a/lib/gpt_providers/openai_gpt_provider.py b/lib/gpt_providers/openai_gpt_provider.py
index c966dad0..0fae454c 100644
--- a/lib/gpt_providers/openai_gpt_provider.py
+++ b/lib/gpt_providers/openai_gpt_provider.py
@@ -17,14 +17,8 @@ import openai
from openai import OpenAI
from pytube import YouTube
import tempfile
-from html2image import Html2Image
import datetime
from PIL import Image
-import moviepy.editor as mp
-import requests
-from moviepy.editor import AudioFileClip
-from concurrent.futures import ThreadPoolExecutor
-
from loguru import logger
logger.remove()
diff --git a/requirements.txt b/requirements.txt
index bcf91212..867554cc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,146 +1,25 @@
-openai
-PyInquirer
+requests
+typer[all]
rich
+python-dotenv
+loguru
+openai
google.generativeai
mistralai
+tenacity
+tavily-python
tabulate
metaphor_python
exa_py
GoogleNews
-sklearn
+clint
+scikit-learn
matplotlib
plotly
requests_html
pytrends
-wordcloud
-rich
-aiofiles
-typer[all]
-aiohttp
-aiosignal
-annotated-types
-anyio
-args
-async-timeout
-asyncio
-attrs
-beautifulsoup4
-blinker
-blis
-Brotli
-catalogue
-certifi
-cffi
-charset-normalizer
-chromedriver-autoinstaller
-click
-clint
-cloudpathlib
-colorama
-confection
-cssselect2
-cymem
-dataclasses-json
-decorator
-distro
-docopt
-duckduckgo-search
-exceptiongroup
-fonttools
-frozenlist
-greenlet
-grpcio
-grpcio-tools
-h11
-h2
-hpack
-html2image
-html5lib
-httpcore
-httpx
-hyperframe
-idna
-imageio
-imageio-ffmpeg
-itsdangerous
-Jinja2
-joblib
-jsonpatch
-jsonpointer
-langchain
-langchain-core
-langcodes
-langsmith
-loguru
-lxml
-Markdown
-markdown2
-MarkupSafe
-marshmallow
-md2pdf
-moviepy
-multidict
-murmurhash
-mypy-extensions
-nltk
-numpy
-openai
-outcome
-packaging
-param
-permchain
-Pillow
-playwright
-preshed
-proglog
-protobuf
-pycparser
-pydub
-pydyf
-pyee
-pyphen
-PySocks
-python-dotenv
-python-multipart
pytube
-PyYAML
-regex
-requests
-selenium
-serpapi
-six
-smart-open
-sniffio
-socksio
-sortedcontainers
-soupsieve
-spacy-legacy
-spacy-loggers
-SQLAlchemy
-srsly
-stability-sdk
-starlette
-tavily-python
-tenacity
-thinc
-tiktoken
-tinycss2
-tqdm
-trio
-trio-websocket
-typer
-typing-inspect
-typing_extensions
-urllib3
-uvicorn
-wasabi
-weasel
-weasyprint
-webdriver-manager
-webencodings
-websocket-client
-Werkzeug
-wsproto
-yarl
-youtube-transcript-api
-zopfli
+wordcloud
+nltk
+prompt_toolkit
+ipython
diff --git a/workspace/ai_stratups.csv b/workspace/ai_stratups.csv
deleted file mode 100644
index 6c63b79f..00000000
--- a/workspace/ai_stratups.csv
+++ /dev/null
@@ -1,5 +0,0 @@
-ο»ΏCompany,URL,Focus Areas,keyword
-Codiga,https://www.codiga.io/,Coding,Code Snippets and Code Analysis
-Mutable AI,https://mutable.ai/,Coding,Build fast with production quality using AI
-Replit Ghostwriter,https://replit.com/,Coding,Accelerate your coding with AI assistance and mobile app
-Stenography,https://stenography.dev/,Coding,Finally. Automatic Documentation.
diff --git a/workspace/arxiv_papers_url.txt b/workspace/arxiv_papers_url.txt
deleted file mode 100644
index d8b7a1a9..00000000
--- a/workspace/arxiv_papers_url.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-https://arxiv.org/abs/1910.10683
-https://arxiv.org/abs/2306.03438
-https://arxiv.org/pdf/2302.06144.pdf
-https://arxiv.org/pdf/2303.03004v3.pdf
-https://arxiv.org/abs/2001.00059
-https://arxiv.org/abs/2012.07023
-https://arxiv.org/abs/2105.08645
-https://arxiv.org/abs/2105.04297
-https://arxiv.org/abs/2010.03150
-https://arxiv.org/abs/2105.12485
-https://arxiv.org/abs/2010.07987
-https://arxiv.org/pdf/2306.13549.pdf
-https://arxiv.org/pdf/2312.16602.pdf
-https://arxiv.org/pdf/2310.03744.pdf
-https://arxiv.org/abs/2312.06647
-https://arxiv.org/pdf/2312.03700.pdf
-https://arxiv.org/abs/2312.09237
-https://arxiv.org/abs/2312.13286
-https://arxiv.org/pdf/2310.20550.pdf
-
diff --git a/workspace/github_topics b/workspace/github_topics
deleted file mode 100644
index 964c18e6..00000000
--- a/workspace/github_topics
+++ /dev/null
@@ -1 +0,0 @@
-image-generation,txt2img,img2img,image2image,text2image,diffusion,generative-art,stability-ai,stable-diffusion,ai,ai-tools,ai-assistant,ai-agents-framework,llm,multi-agent,agent,llama2,mistral,fine-tuning,rag,generative,prompt-engineering,prompt-tuning,generative-ai,text-to-image-generation,llm-ops,retrieval-augmented-generation,langchain,gemini-api,vertex-ai,huggingface,semantic-search,auto-gpt,llmops,ai-toolkit,chatbot,chatgpt,chat-gpt,multimodal,code-assistant,text-to-video,llms,gpt-4
diff --git a/workspace/github_url_to_write.csv b/workspace/github_url_to_write.csv
deleted file mode 100644
index 6e980ee8..00000000
--- a/workspace/github_url_to_write.csv
+++ /dev/null
@@ -1,135 +0,0 @@
-https://github.com/Significant-Gravitas/AutoGPT
-https://github.com/gpt-engineer-org/gpt-engineer
-https://github.com/reworkd/AgentGPT
-https://github.com/geekan/MetaGPT
-https://github.com/Josh-XT/AGiXT
-https://github.com/litanlitudan/skyagi
-https://github.com/joonspk-research/generative_agents
-https://github.com/smol-ai/developer
-https://github.com/Forethought-Technologies/AutoChain
-https://github.com/TransformerOptimus/SuperAGI
-https://github.com/homanp/superagent
-https://github.com/a16z-infra/ai-town
-https://github.com/AI-Engineer-Foundation/agent-protocol
-https://github.com/microsoft/autogen
-https://github.com/cpacker/MemGPT
-https://github.com/shroominic/codeinterpreter-api
-https://github.com/aiwaves-cn/agents
-https://github.com/dataelement/bisheng
-https://github.com/Maplemx/Agently
-https://github.com/zilliztech/GPTCache
-http://github.com//Significant-Gravitas/AutoGPT
-http://github.com//AUTOMATIC1111/stable-diffusion-webui
-http://github.com//gpt-engineer-org/gpt-engineer
-http://github.com//lencx/ChatGPT
-http://github.com//hpcaitech/ColossalAI
-http://github.com//LAION-AI/Open-Assistant
-http://github.com//xitu/gold-miner
-http://github.com//babysor/MockingBird
-http://github.com//google-research/google-research
-http://github.com//photoprism/photoprism
-http://github.com//explosion/spaCy
-http://github.com//AMAI-GmbH/AI-Expert-Roadmap
-http://github.com//StanGirard/quivr
-http://github.com//microsoft/AI-For-Beginners
-http://github.com//GitHubDaily/GitHubDaily
-http://github.com//Lightning-AI/pytorch-lightning
-http://github.com//lutzroeder/netron
-http://github.com//JushBJJ/Mr.-Ranedeer-AI-Tutor
-http://github.com//s0md3v/roop
-http://github.com//microsoft/generative-ai-for-beginners
-http://github.com//leon-ai/leon
-http://github.com//geekan/MetaGPT
-http://github.com//jmorganca/ollama
-http://github.com//run-llama/llama_index
-http://github.com//milvus-io/milvus
-http://github.com//chatchat-space/Langchain-Chatchat
-http://github.com//zhayujie/chatgpt-on-wechat
-http://github.com//mindsdb/mindsdb
-http://github.com//FlowiseAI/Flowise
-http://github.com//microsoft/unilm
-http://github.com//mlabonne/llm-course
-http://github.com//microsoft/semantic-kernel
-http://github.com//ymcui/Chinese-LLaMA-Alpaca
-http://github.com//mudler/LocalAI
-http://github.com//mlc-ai/mlc-llm
-http://github.com//THUDM/ChatGLM2-6B
-http://github.com//langgenius/dify
-http://github.com//vllm-project/vllm
-http://github.com//TransformerOptimus/SuperAGI
-http://github.com//ludwig-ai/ludwig
-http://github.com//hiyouga/LLaMA-Factory
-http://github.com//bentoml/OpenLLM
-http://github.com//cloneofsimo/lora
-http://github.com//eosphoros-ai/DB-GPT
-http://github.com//labring/FastGPT
-http://github.com//Mintplex-Labs/anything-llm
-http://github.com//danswer-ai/danswer
-http://github.com//neuml/txtai
-http://github.com//run-llama/rags
-http://github.com//postgresml/postgresml
-http://github.com//h2oai/h2ogpt
-http://github.com//css-doodle/css-doodle
-http://github.com//williamngan/pts
-http://github.com//dair-ai/Prompt-Engineering-Guide
-http://github.com//AI4Finance-Foundation/FinGPT
-http://github.com//yzfly/awesome-chatgpt-zh
-http://github.com//microsoft/promptflow
-http://github.com//jina-ai/jina
-http://github.com//deepset-ai/haystack
-http://github.com//open-mmlab/mmagic
-http://github.com//bentoml/BentoML
-http://github.com//openvinotoolkit/openvino
-http://github.com//reworkd/AgentGPT
-http://github.com//logspace-ai/langflow
-http://github.com//mayooear/gpt4-pdf-chatbot-langchain
-http://github.com//botpress/botpress
-http://github.com//activeloopai/deeplake
-http://github.com//danny-avila/LibreChat
-http://github.com//liaokongVFX/LangChain-Chinese-Getting-Started-Guide
-http://github.com//kyrolabs/awesome-langchain
-http://github.com//zilliztech/GPTCache
-http://github.com//speechbrain/speechbrain
-http://github.com//vercel/ai
-http://github.com//skorch-dev/skorch
-http://github.com//baichuan-inc/Baichuan-7B
-http://github.com//microsoft/autogen
-http://github.com//f/awesome-chatgpt-prompts
-http://github.com//xtekky/gpt4free
-http://github.com//python-telegram-bot/python-telegram-bot
-http://github.com//wechaty/wechaty
-http://github.com//RasaHQ/rasa
-http://github.com//lobehub/lobe-chat
-http://github.com//transitive-bullshit/chatgpt-api
-http://github.com//GaiZhenbiao/ChuanhuChatGPT
-http://github.com//gunthercox/ChatterBot
-http://github.com//mamoe/mirai
-http://github.com//haotian-liu/LLaVA
-http://github.com//howdyai/botkit
-http://github.com//databrickslabs/dolly
-http://github.com//chiphuyen/stanford-tensorflow-tutorials
-http://github.com//ChatGPTNextWeb/ChatGPT-Next-Web
-http://github.com//openai/openai-cookbook
-http://github.com//binary-husky/gpt_academic
-http://github.com//PlexPt/awesome-chatgpt-prompts-zh
-http://github.com//KillianLucas/open-interpreter
-http://github.com//acheong08/ChatGPT
-http://github.com//tw93/Pake
-http://github.com//LC044/WeChatMsg
-http://github.com//openai/chatgpt-retrieval-plugin
-http://github.com//openai-translator/openai-translator
-http://github.com//sweepai/sweep
-http://github.com//lucidrains/imagen-pytorch
-http://github.com//GokuMohandas/Made-With-ML
-http://github.com//TabbyML/tabby
-http://github.com//chroma-core/chroma
-http://github.com//eugeneyan/open-llms
-http://github.com//cleanlab/cleanlab
-http://github.com//RUCAIBox/LLMSurvey
-http://github.com//OpenNMT/OpenNMT-py
-http://github.com//joaomdmoura/crewAI
-http://github.com//Pythagora-io/gpt-pilot
-http://github.com//mouredev/Hello-Python
-http://github.com//Bin-Huang/chatbox
-http://github.com//getumbrel/llama-gpt
-http://github.com//gventuri/pandas-ai
diff --git a/workspace/github_urls_to_research_yet b/workspace/github_urls_to_research_yet
deleted file mode 100644
index cce7ce05..00000000
--- a/workspace/github_urls_to_research_yet
+++ /dev/null
@@ -1,9 +0,0 @@
-https://github.com/louisfb01/best_AI_papers_2023
-https://github.com/Giskard-AI/awesome-ai-safety
-https://github.com/mahseema/awesome-ai-tools
-https://github.com/Hyraze/ai-collective-tools#image-generator
-https://github.com/Horhorist/Awesome-ai
-https://github.com/youraibot/AI-Toolkit
-https://github.com/hades217/awesome-ai
-https://github.com/WooooDyy/LLM-Agent-Paper-List
-https://github.com/e2b-dev/awesome-ai-agents
diff --git a/workspace/keyword_blog.gif b/workspace/keyword_blog.gif
new file mode 100644
index 00000000..89946c76
Binary files /dev/null and b/workspace/keyword_blog.gif differ
diff --git a/workspace/keyword_blog.png b/workspace/keyword_blog.png
new file mode 100644
index 00000000..19d714f3
Binary files /dev/null and b/workspace/keyword_blog.png differ