kunthawat/ALwrity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP- Try AI-Writer and Web research; working. Working on usuability aspects.

Browse Source
This commit is contained in:
AjaySi
2024-03-02 08:52:42 +05:30
parent a87a87a620
commit 87053406ba
14 changed files with 162 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
lib/ai_web_researcher/google_trends_researcher.py
View File

@@ -182,7 +182,10 @@ def get_related_topics_and_save_csv(search_keywords):
pytrends.build_payload(search_keywords, cat=0, timeframe='today 12-m')
# Get related topics
data = pytrends.related_topics()
try:
data = pytrends.related_topics()
except Exception as err:
logger.error(f"Failed to get pytrends realted topics: {err}")
# Extract data from the result
top_topics = list(data.values())[0]['top']
rising_topics = list(data.values())[0]['rising']
@@ -215,7 +218,7 @@ def get_related_topics_and_save_csv(search_keywords):
return all_topics_df
except Exception as e:
print(f"ERROR: An error occurred in related topics: {e}")
logger.error(f"ERROR: An error occurred in related topics: {e}")
return pd.DataFrame()

13
lib/ai_web_researcher/gpt_online_researcher.py
View File

@@ -47,7 +47,9 @@ def do_google_serp_search(search_keywords):
""" """
try:
logger.info(f"Doing Google search for: {search_keywords}\n")
return(google_search(search_keywords))
g_results = google_search(search_keywords)
g_titles = extract_info(g_results, 'titles')
return(g_results, g_titles)
except Exception as err:
logger.error(f"Failed to do Google Serpapi research: {err}")
# Not failing, as tavily would do same and then GPT-V to search.
@@ -58,7 +60,9 @@ def do_tavily_ai_search(search_keywords, include_domains=None):
try:
# FIXME: Include the follow-up questions as blog FAQs.
logger.info(f"Doing Tavily AI search for: {search_keywords}")
return(get_tavilyai_results(search_keywords, include_domains))
t_results = get_tavilyai_results(search_keywords, include_domains)
t_titles = tavily_extract_information(t_results, 'titles')
return(t_results, t_titles)
except Exception as err:
logger.error(f"Failed to do Tavily AI Search: {err}")
@@ -75,7 +79,8 @@ def do_metaphor_ai_research(search_keywords,
include_domains=include_domains,
time_range=time_range,
similar_url=similar_url)
return response_articles
m_titles = metaphor_extract_titles_or_text(response_articles, return_titles=True)
return(response_articles, m_titles)
except Exception as err:
logger.error(f"Failed to do Metaphor search: {err}")
@@ -139,7 +144,7 @@ def tavily_extract_information(json_data, keyword):
Returns:
list or str: The extracted information based on the keyword.
"""
if keyword == 'title':
if keyword == 'titles':
return [result['title'] for result in json_data['results']]
elif keyword == 'content':
return [result['content'] for result in json_data['results']]

5
lib/ai_writers/combine_blog_and_keywords.py
View File

@@ -22,12 +22,9 @@ def blog_with_keywords(blog, keywords):
prompt = f"""
You are an expert copywriter specializing in content optimization for SEO.
I will provide you with my 'blog content' and 'list of keywords' on the same topic.
Your task is to write an original blog, using the given keywords and blog content.
Your task is to write an original blog, using given keywords and blog content.
Your blog should be highly detailed and well formatted.
Do not miss out any details from provided blog content.
Always, include figures, data, results from given content.
It is important that your blog is original and unique. It should be highly readable and SEO optimized.
Blog content: '{blog}'
list of keywords: '{keywords}'

61
lib/ai_writers/keywords_to_blog.py
View File

@@ -18,9 +18,13 @@ from .blog_from_google_serp import write_blog_google_serp
from .combine_research_and_blog import blog_with_research
from .combine_blog_and_keywords import blog_with_keywords
from ..ai_web_researcher.you_web_reseacher import get_rag_results, search_ydc_index
from ..blog_metadata.get_blog_metadata import blog_metadata
from ..blog_postprocessing.save_blog_to_file import save_blog_to_file
from ..blog_postprocessing.blog_proof_reader import blog_proof_editor
def write_blog_from_keywords(search_keywords, url=None, output_format="markdown"):
def write_blog_from_keywords(search_keywords, url=None):
"""
This function will take a blog Topic to first generate sections for it
and then generate content for each section.
@@ -31,50 +35,51 @@ def write_blog_from_keywords(search_keywords, url=None, output_format="markdown"
logger.info(f"Researching and Writing Blog on keywords: {search_keywords}")
# Use to store the blog in a string, to save in a *.md file.
blog_markdown_str = ""
example_blog_titles = []
# Call on the got-researcher, tavily apis for this. Do google search for organic competition.
google_search_result = do_google_serp_search(search_keywords)
google_search_result, g_titles = do_google_serp_search(search_keywords)
example_blog_titles.append(g_titles)
blog_markdown_str = write_blog_google_serp(search_keywords, google_search_result)
# logger.info/check the final blog content.
logger.info(f"Final blog content: {blog_markdown_str}")
# Do Tavily AI research to augument the above blog.
tavily_search_result = do_tavily_ai_search(search_keywords)
tavily_search_result, t_titles = do_tavily_ai_search(search_keywords)
example_blog_titles.append(t_titles)
blog_markdown_str = blog_with_research(blog_markdown_str, tavily_search_result)
logger.info(f"Final blog content: {blog_markdown_str}")
# Do Metaphor/Exa AI search.
metaphor_search_result = do_metaphor_ai_research(search_keywords)
blog_markdown_str = blog_with_research(blog_markdown_str, metaphor_search_result)
logger.info(f"Final blog content: {blog_markdown_str}")
try:
# Do Metaphor/Exa AI search.
metaphor_search_result, m_titles = do_metaphor_ai_research(search_keywords)
example_blog_titles.append(m_titles)
blog_markdown_str = blog_with_research(blog_markdown_str, metaphor_search_result)
logger.info(f"Final blog content: {blog_markdown_str}")
except Exception as err:
logger.error(f"Failed to do Metaphor AI search: {err}")
# Do Google trends analysis and combine with latest blog.
pytrends_search_result = do_google_pytrends_analysis(search_keywords)
blog_markdown_str = blog_with_keywords(blog_markdown_str, pytrends_search_result)
try:
pytrends_search_result = do_google_pytrends_analysis(search_keywords)
blog_markdown_str = blog_with_keywords(blog_markdown_str, pytrends_search_result)
except Exception as err:
logger.error(f"Failed to do Google Trends Analysis:{err}")
blog_markdown_str = blog_proof_editor(blog_markdown_str, search_keywords)
logger.info(f"Final blog content: {blog_markdown_str}")
# Combine YOU.com RAG search with the latest blog content.
#you_rag_result = get_rag_results(search_keywords)
you_search_result = search_ydc_index(search_keywords)
blog_markdown_str = blog_with_research(blog_markdown_str, you_search_result)
logger.info(f"Final blog content: {blog_markdown_str}")
#you_search_result = search_ydc_index(search_keywords)
#blog_markdown_str = blog_with_research(blog_markdown_str, you_search_result)
#logger.info(f"Final blog content: {blog_markdown_str}")
exit(1)
blog_title = generate_blog_title(blog_markdown_str, "gemini")
blog_meta_desc = generate_blog_description(blog_markdown_str, "gemini")
logger.info(f"The blog meta description is: {blog_meta_desc}\n")
blog_tags = get_blog_tags(blog_markdown_str, "gemini")
logger.info(f"Blog tags for generated content: {blog_tags}")
blog_categories = get_blog_categories(blog_markdown_str, "gemini")
logger.info(f"Generated blog categories: {blog_categories}\n")
#blog_markdown_str = gemini_get_code_samples(blog_markdown_str)
#logger.info(f"Blog with code sample: \n {blog_markdown_str}")
blog_title, blog_meta_desc, blog_tags, blog_categories = blog_metadata(blog_markdown_str,
search_keywords, example_blog_titles)
# fixme: Remove the hardcoding, need add another option OR in config ?
image_dir = os.path.join(os.getcwd(), "blog_images")
generated_image_name = f"generated_image_{datetime.datetime.now():%Y-%m-%d-%H-%M-%S}.png"
generated_image_name = f"generated_image_{datetime.now():%Y-%m-%d-%H-%M-%S}.png"
generated_image_filepath = os.path.join(image_dir, generated_image_name)
# Generate an image based on meta description
#logger.info(f"Calling Image generation with prompt: {blog_meta_desc}")
@@ -87,4 +92,4 @@ def write_blog_from_keywords(search_keywords, url=None, output_format="markdown"
# TBD: Save the blog content as a .md file. Markdown or HTML ?
save_blog_to_file(blog_markdown_str, blog_title, blog_meta_desc, blog_tags, blog_categories, generated_image_filepath)
logger.info(f"\n\n ################ Finished writing Blog for : {akeyword} #################### \n")
logger.info(f"\n\n ################ Finished writing Blog for : {search_keywords} #################### \n")

15
lib/blog_metadata/get_blog_category.py
View File

@@ -1,8 +1,9 @@
import sys
import os
from .gpt_providers.openai_gpt_provider import openai_chatgpt
from .gpt_providers.gemini_pro_text import gemini_text_response
from pathlib import Path
from dotenv import load_dotenv
load_dotenv(Path('../.env'))
from loguru import logger
logger.remove()
logger.add(sys.stdout,
@@ -10,11 +11,15 @@ logger.add(sys.stdout,
format="<level>{level}</level>|<green>{file}:{line}:{function}</green>| {message}"
)
from ..gpt_providers.openai_gpt_provider import openai_chatgpt
from ..gpt_providers.gemini_pro_text import gemini_text_response
def get_blog_categories(blog_article, gpt_providers):
def get_blog_categories(blog_article):
"""
Function to generate blog categories for given blog content.
"""
gpt_providers = os.environ["GPT_PROVIDER"]
prompt = f"""As an expert SEO and content writer, I will provide you with blog content.
Suggest only 2 blog categories which are most relevant to provided blog content,
by identifying the main topic. Also consider the target audience and the
@@ -22,7 +27,7 @@ def get_blog_categories(blog_article, gpt_providers):
The blog content is: '{blog_article}'"
"""
logger.info("Generating blog categories for the given blog.")
if 'gemini' in gpt_providers:
if 'google' in gpt_providers:
try:
response = gemini_text_response(prompt)
return response

13
lib/blog_metadata/get_blog_meta_desc.py
View File

@@ -1,11 +1,6 @@
import sys
import os
from pathlib import Path
from .gpt_providers.openai_chat_completion import openai_chatgpt
from .gpt_providers.gemini_pro_text import gemini_text_response
import google.generativeai as genai
from dotenv import load_dotenv
load_dotenv(Path('../.env'))
@@ -16,11 +11,15 @@ logger.add(sys.stdout,
format="<level>{level}</level>|<green>{file}:{line}:{function}</green>| {message}"
)
from ..gpt_providers.openai_chat_completion import openai_chatgpt
from ..gpt_providers.gemini_pro_text import gemini_text_response
def generate_blog_description(blog_content, gpt_providers):
def generate_blog_description(blog_content):
"""
Prompt designed to give SEO optimized blog descripton
"""
gpt_providers = os.environ["GPT_PROVIDER"]
logger.info("Generating Blog Meta Description for the given blog.")
prompt = f"""As an expert SEO and blog writer, Compose a compelling meta description for the given blog content,
adhering to SEO best practices. Keep it between 150-160 characters.
@@ -28,7 +27,7 @@ def generate_blog_description(blog_content, gpt_providers):
Respond with only one of your best effort and do not include your explanations.
Blog Content: '{blog_content}'"""
if 'gemini' in gpt_providers:
if 'google' in gpt_providers:
try:
response = gemini_text_response(prompt)
return response

10
lib/blog_metadata/get_blog_metadata.py
View File

@@ -13,14 +13,14 @@ logger.add(sys.stdout,
)
def blog_metadata(blog_content, gpt_providers="openai"):
def blog_metadata(blog_content, search_keywords, blog_titles):
""" Common function to get blog metadata """
blog_title = generate_blog_title(blog_content, gpt_providers)
blog_meta_desc = generate_blog_description(blog_content, gpt_providers)
blog_title = generate_blog_title(blog_content, search_keywords, blog_titles)
blog_meta_desc = generate_blog_description(blog_content)
logger.info(f"The blog meta description is: {blog_meta_desc}\n")
blog_tags = get_blog_tags(blog_content, gpt_providers)
blog_tags = get_blog_tags(blog_content)
logger.info(f"Blog tags for generated content: {blog_tags}")
blog_categories = get_blog_categories(blog_content, gpt_providers)
blog_categories = get_blog_categories(blog_content)
logger.info(f"Generated blog categories: {blog_categories}\n")
return(blog_title, blog_meta_desc, blog_tags, blog_categories)

42
lib/blog_metadata/get_blog_title.py
View File

@@ -1,7 +1,12 @@
import os
import sys
from .gpt_providers.openai_chat_completion import openai_chatgpt
from .gpt_providers.gemini_pro_text import gemini_text_response
from pathlib import Path
from dotenv import load_dotenv
load_dotenv(Path('../../.env'))
from ..gpt_providers.openai_chat_completion import openai_chatgpt
from ..gpt_providers.gemini_pro_text import gemini_text_response
from loguru import logger
logger.remove()
@@ -11,20 +16,33 @@ logger.add(sys.stdout,
)
def generate_blog_title(blog_article, gpt_providers="openai"):
def generate_blog_title(blog_article, keywords=None, example_titles=None, num_titles=1):
"""
Given a blog title generate an outline for it
"""
prompt = ''
gpt_providers = os.environ["GPT_PROVIDER"]
logger.info("Generating blog title.")
prompt = f"""As a SEO expert, I will provide you with a blog content.
Your task is write a SEO optimized, call to action and engaging blog title for it.
Follows SEO best practises to suggest the blog title.
Please keep the titles concise, not exceeding 60 words.
Respond with only one title and no explanations.
Important: Your response should be in plaintext.
Generate blog title for this given blog content:\n '{blog_article}' """
if 'gemini' in gpt_providers:
if not keywords and not example_titles:
prompt = f"""As a SEO expert, I will provide you with a blog content.
Your task is write a SEO optimized and call to action, blog title for given blog content.
Follow SEO best practises to suggest the blog title.
Please keep the titles concise, not exceeding 60 words.
Respond with only {num_titles} title and no explanations.
Negative Keywords: Unvieling, unleash, power of. Dont use such words in your title.
Generate {num_titles} blog title for this given blog content:\n '{blog_article}' """
elif keywords and example_titles:
prompt = f"""As a SEO expert, I will provide you with my blog keywords and example titles.
Your task is to write {num_titles} blog title.
Ensure that your blog titles will help in competing against given example titles.
Follow SEO best practises to suggest the blog title.
Please keep the titles concise, not exceeding 60 words.
Respond with only {num_titles} title and no explanations.
Negative Keywords: Unvieling, unleash, power of. Dont use such words in your title.
Blog Keywords: '{keywords}'
Example Titles: '{example_titles}'
"""
if 'google' in gpt_providers:
try:
response = gemini_text_response(prompt)
return response

15
lib/blog_metadata/get_tags.py
View File

@@ -1,8 +1,9 @@
import sys
import os
from .gpt_providers.openai_chat_completion import openai_chatgpt
from .gpt_providers.gemini_pro_text import gemini_text_response
from pathlib import Path
from dotenv import load_dotenv
load_dotenv(Path('../.env'))
from loguru import logger
logger.remove()
logger.add(sys.stdout,
@@ -10,17 +11,21 @@ logger.add(sys.stdout,
format="<level>{level}</level>|<green>{file}:{line}:{function}</green>| {message}"
)
from ..gpt_providers.openai_chat_completion import openai_chatgpt
from ..gpt_providers.gemini_pro_text import gemini_text_response
def get_blog_tags(blog_article, gpt_providers):
def get_blog_tags(blog_article):
"""
Function to suggest tags for the given blog content
"""
# Suggest at least 5 tags for the following blog post [Enter your blog post text here].
gpt_providers = os.environ["GPT_PROVIDER"]
prompt = f"""As an expert SEO and blog writer, suggest only 2 relevant and specific blog tags
for the given blog content. Only reply with comma separated values.
Blog content: {blog_article}."""
logger.info("Generating Blog tags for the given blog post.")
if 'gemini' in gpt_providers:
if 'google' in gpt_providers:
try:
response = gemini_text_response(prompt)
return response

45
lib/blog_postprocessing/blog_proof_reader.py
View File

@@ -1,33 +1,36 @@
from .gpt_providers.gemini_pro_text import gemini_text_response
from .gpt_providers.openai_chat_completion import openai_chatgpt
import os
import sys
from pathlib import Path
from dotenv import load_dotenv
load_dotenv(Path('../../.env'))
from ..gpt_providers.gemini_pro_text import gemini_text_response
from ..gpt_providers.openai_chat_completion import openai_chatgpt
def blog_proof_editor(blog_content, blog_keywords, gpt_provider="openai"):
def blog_proof_editor(blog_content, blog_keywords):
"""
Helper for blog proof reading.
"""
prompt = f"""I am looking for detailed editing and enhancement of the given blog post,
with a particular focus on maintaining originality.
The topic of the content is [{blog_keywords}]. Please go through the blog and make direct edits to improve it,
ensuring the final output is both high-quality and original.
Note: There are duplicates headings and corresponding paragraphs, rewrite into one subheading.
Here are the specific areas to focus on:
gpt_provider = os.environ["GPT_PROVIDER"]
prompt = f"""As an expert copywriter, I will provide you with 'my blog' and its 'main keywords'.
Your task is to rewrite my blog, by following the guidelines below.
Below are the guidelines to follow:
1). Ensure Originality: Edit any sections that lack originality, replacing them with unique and creative content.
2). Eliminate Repetitive Language: Rewrite repetitive phrases with varied and engaging language.
3). Vocabulary and Grammar Enhancement: Directly correct any grammatical errors and upgrade the
2). Vocabulary and Grammar Enhancement: Directly correct any grammatical errors and upgrade the
vocabulary for better readability.
4). Improve Sentence Structure: Enhance sentence construction for better clarity and flow.
5). Tone and Brand Alignment: Adjust the tone, voice, personality of given content to make it unique.
6). Optimize Content Structure: Reorganize the content for a more impactful presentation,
3). Improve Sentence Structure: Enhance sentence construction for better clarity and flow.
4). Tone and Brand Alignment: Adjust the tone, voice, personality of given content to make it unique.
5). Optimize Content Structure: Reorganize the content for a more impactful presentation,
including better paragraphing and transitions.
7). Remove Redundancies: Important, Cut out any redundant information or overly complex jargon.
8). Refine Overall Structure: Make structural changes to improve the overall impact of the content.
9). Remember, rewrite all content that repeated, while maintaining the formatting of the given blog text.
6). Simplify given content: Simplify concepts and replace overly complex jargons and words.
7). Refine Overall Structure: Make structural changes to improve the overall impact of the content.
Please apply these changes directly to the following blog post and provide the edited version:\n
'{blog_content}'. """
\n\nMain keywords: '{blog_keywords}'
My Blog: '{blog_content}'. """
if 'openai' in gpt_provider:
try:
@@ -35,7 +38,7 @@ def blog_proof_editor(blog_content, blog_keywords, gpt_provider="openai"):
return response
except Exception as err:
SystemError(f"Openai Error Blog Proof Reading: {err}")
elif 'gemini' in gpt_provider:
elif 'google' in gpt_provider:
try:
response = gemini_text_response(prompt)
return response

1
lib/gpt_providers/openai_gpt_provider.py
View File

@@ -25,7 +25,6 @@ import requests
from moviepy.editor import AudioFileClip
from concurrent.futures import ThreadPoolExecutor
from ..gpt_online_researcher import do_online_research
from loguru import logger
logger.remove()