kunthawat/ALwrity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AI Agents content ideator team, prompt config

Browse Source
This commit is contained in:
ajaysi
2024-05-20 15:07:20 +05:30
parent b431bfcbd8
commit 8f2bf02b65
10 changed files with 172 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
lib/ai_web_researcher/google_trends_researcher.py
View File

@@ -331,10 +331,11 @@ def get_suggestions_for_keyword(search_term):
pd.set_option('display.max_rows', expanded_results_df.shape[0]+1)
expanded_results_df.drop_duplicates('Keywords', inplace=True)
table = tabulate(expanded_results_df, headers=['Keywords', 'Relevance'], tablefmt='fancy_grid')
try:
save_in_file(table)
except Exception as save_results_err:
logger.error(f"Failed to save search results: {save_results_err}")
# FIXME: Too much data for LLM context window. We will need to embed it.
#try:
# save_in_file(table)
#except Exception as save_results_err:
# logger.error(f"Failed to save search results: {save_results_err}")
return expanded_results_df
except Exception as e:
logger.error(f"get_suggestions_for_keyword: Error in main: {e}")

4
lib/ai_web_researcher/gpt_online_researcher.py
View File

@@ -67,12 +67,12 @@ def do_google_serp_search(search_keywords):
# Not failing, as tavily would do same and then GPT-V to search.
def do_tavily_ai_search(search_keywords):
def do_tavily_ai_search(search_keywords, max_results=10):
""" Common function to do Tavily AI web research."""
try:
# FIXME: Include the follow-up questions as blog FAQs.
logger.info(f"Doing Tavily AI search for: {search_keywords}")
t_results = get_tavilyai_results(search_keywords)
t_results = get_tavilyai_results(search_keywords, max_results)
t_titles = tavily_extract_information(t_results, 'titles')
t_answer = tavily_extract_information(t_results, 'answer')
return(t_results, t_titles, t_answer)

6
lib/ai_web_researcher/tavily_ai_search.py
View File

@@ -49,7 +49,7 @@ from tenacity import retry, stop_after_attempt, wait_random_exponential
@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
def get_tavilyai_results(keywords):
def get_tavilyai_results(keywords, max_results=10):
"""
Get Tavily AI search results based on specified keywords and options.
@@ -86,13 +86,13 @@ def get_tavilyai_results(keywords):
tavily_search_result = client.search(keywords,
search_depth="advanced",
include_answer=True,
max_results=10,
max_results=max_results,
include_domains=include_urls)
else:
tavily_search_result = client.search(keywords,
search_depth = "advanced",
include_answer=True,
max_results=10)
max_results=max_results)
print_result_table(tavily_search_result)
return(tavily_search_result)

10
lib/ai_writers/ai_agents_crew_writer.py
View File

@@ -16,7 +16,7 @@ def create_agents(search_keywords):
# Set gemini pro as llm
llm = ChatGoogleGenerativeAI(
model="gemini-pro", verbose=True, temperature=0.6, google_api_key=google_api_key
model="gemini-1.5-flash-latest", verbose=True, temperature=0.6, google_api_key=google_api_key
)
role, goal, backstory = read_config("content_researcher")
@@ -28,7 +28,7 @@ def create_agents(search_keywords):
memory = True, # Enable memory
verbose = True,
max_rpm = None, # No limit on requests per minute
max_iter = 15, # Default value for maximum iterations
max_iter = 10, # Default value for maximum iterations
allow_delegation = False,
llm = llm
)
@@ -42,7 +42,7 @@ def create_agents(search_keywords):
verbose = True,
tools = [search_tool],
max_rpm = 10, # No limit on requests per minute
max_iter = 5, # Default value for maximum iterations
max_iter = 10, # Default value for maximum iterations
allow_delegation = False,
llm = llm
)
@@ -55,7 +55,7 @@ def create_agents(search_keywords):
memory = True, # Enable memory
verbose = True,
max_rpm = 10, # No limit on requests per minute
max_iter = 5, # Default value for maximum iterations
max_iter = 15, # Default value for maximum iterations
allow_delegation = False,
llm = llm
)
@@ -68,7 +68,7 @@ def create_agents(search_keywords):
memory=True, # Enable memory
verbose=True,
max_rpm=10, # No limit on requests per minute
max_iter=5, # Default value for maximum iterations
max_iter=10, # Default value for maximum iterations
allow_delegation=False,
llm=llm
)

65
lib/ai_writers/long_form_ai_writer.py
View File

@@ -6,6 +6,7 @@
#####################################################
import os
import re
import time #iwish
import sys
import yaml
@@ -94,6 +95,7 @@ def long_form_generator(content_keywords):
content_outline = prompts.get('content_outline').format(
content_language=content_language,
content_title='{content_title}',
content_type=content_type,
target_audience=target_audience
)
@@ -115,10 +117,16 @@ def long_form_generator(content_keywords):
# Configure generative AI
load_dotenv(Path('../.env'))
generation_config = {
"temperature": 0.8,
"top_p": 0.95,
"max_output_tokens": 8192,
}
genai.configure(api_key=os.getenv('GEMINI_API_KEY'))
# Initialize the generative model
model = genai.GenerativeModel('gemini-pro')
model_pro = genai.GenerativeModel('gemini-1.5-flash-latest')
model = genai.GenerativeModel('gemini-pro', generation_config=generation_config)
model_pro = genai.GenerativeModel('gemini-1.5-flash-latest', generation_config=generation_config)
# Do SERP web research for given keywords to generate title and outline.
web_research_result, g_titles = do_google_serp_search(content_keywords)
@@ -132,12 +140,27 @@ def long_form_generator(content_keywords):
return
try:
content_outline = generate_with_retry(model_pro,
content_outline.format(content_title=content_title, web_research_result=web_research_result)).text
content_outline = generate_with_retry(model_pro, content_outline.format(
content_title=content_title,
web_research_result=web_research_result)).text
logger.info(f"The content Outline is: {content_outline}\n\n")
except Exception as err:
logger.error(f"Failed to generate content outline: {err}")
try:
logger.info("Do web research with Tavily to provide context for content creation.")
# Do Metaphor/Exa AI search.
table_data = []
web_research_result, m_titles, t_titles = do_tavily_ai_search(content_keywords, max_results=5)
for item in web_research_result.get("results"):
title = item.get("title", "")
snippet = item.get("content", "")
table_data.append([title, snippet])
web_research_result = table_data
except Exception as err:
logger.error(f"Failed to do Tavily AI search: {err}")
return
try:
starting_draft = generate_with_retry(model_pro, starting_prompt.format(
content_title=content_title,
@@ -165,24 +188,27 @@ def long_form_generator(content_keywords):
draft += '\n\n' + continuation
except Exception as err:
logger.error(f"Failed as: {err} and {continuation}")
try:
logger.info("Do web research with Tavily to provide context for content creation.")
# Do Metaphor/Exa AI search.
table_data = []
web_research_result, m_titles, t_titles = do_tavily_ai_search(content_keywords)
for item in web_research_result.get("results"):
title = item.get("title", "")
snippet = item.get("content", "")
table_data.append([title, snippet])
web_research_result = table_data
except Exception as err:
logger.error(f"Failed to do Tavily AI search: {err}")
return
logger.info(f"Writing in progress... Current draft length: {len(draft)} characters")
search_terms = f"""
I will provide you with blog outline, your task is to read the outline & return 3 google search keywords.
Your response will be used to do web research for writing on the given outline.
Do not explain your response, provide 3 google search sentences encompassing the given content outline.
Content Outline:\n\n
{content_outline}
"""
search_words = generate_with_retry(model_pro, search_terms).text
while 'IAMDONE' not in continuation:
try:
web_research_result, m_titles = do_metaphor_ai_research(content_keywords)
#web_research_result, m_titles = do_metaphor_ai_research(content_keywords)
str_list = re.split(r',\s*', search_words)
# Strip quotes from each element
str_list = [s.strip('\'"') for s in str_list]
for search_term in str_list:
web_research_result, m_titles, t_titles = do_tavily_ai_search(search_term, max_results=5)
continuation = generate_with_retry(model, continuation_prompt.format(
content_title=content_title,
content_outline=content_outline,
@@ -195,9 +221,6 @@ def long_form_generator(content_keywords):
# At this point, the context is little stale. We should more web research on
# related queries as per the content outline, to augment the LLM context.
# web_research_result, m_titles = do_metaphor_ai_research(content_keywords)
#logger.info(f"Doing Tavily Search Again, Should mix with Exa.ai")
#web_research_result, m_titles, t_titles = do_tavily_ai_search(content_title)
except Exception as err:
logger.error(f"Failed to continually write the Essay: {err}")
return

101
lib/content_planning_calender/content_planning_agents_alwrity_crew.py
View File

@@ -8,10 +8,7 @@ from crewai_tools import FileReadTool
from ..ai_web_researcher.google_trends_researcher import do_google_trends_analysis
def setup_environment():
os.environ["OPENAI_MODEL_NAME"] = 'gpt-3.5-turbo' # Adjust based on given model
def create_agents(search_keywords):
def create_agents(search_keywords, already_written_on):
# Tools for the agents.
search_tool = SerperDevTool()
@@ -26,7 +23,7 @@ def create_agents(search_keywords):
file_read_tool = FileReadTool(file_path=os.getenv('SEARCH_SAVE_FILE'))
# The manager keeps an eye on the content already planned to give new ideas.
# TBD: Accept the user website urls and populate the file with sitemap.xml
manager_read_tool = FileReadTool(file_path="content_already_planned.txt")
manager_read_tool = FileReadTool(file_path=already_written_on)
# Load the google gemini api key
google_api_key = os.getenv("GEMINI_API_KEY")
@@ -38,9 +35,8 @@ def create_agents(search_keywords):
content_researcher = Agent(
role = 'Senior Web Research Analyst (Content Strategy): Aisha Sharma',
goal = f"""
Recommend unique blog titles for "{search_keywords}" keywords.
Provide researched titles to be used for content calender & planning.""",
goal = f"""Help Create a highly detailed 2 month-long content calender, focused around keywords: {search_keywords}.
Provide web researched titles to be used for content calender & planning to Ted XingPi""",
backstory = f"""
Your Focus: Content Opportunity Analysis & Keyword Research ({search_keywords}).
@@ -50,16 +46,13 @@ def create_agents(search_keywords):
1). Web Research & Content Gap Identification (Expert).
2). SEO Best Practices, Keyword Research & content planning expert (Advanced).
3). Analyzes search trends and competitor content.
Mission:
1). Fuel company's content strategy with data-driven insights to attract and educate online readers.
2). Identifies high-volume, low-competition keywords relevant to {search_keywords}.
4). Fuel company's content strategy with data-driven insights to attract and educate online readers.
5). Identifies high-volume, low-competition keywords relevant to {search_keywords}.
Responsibilities:
1). Recommend high-value content opportunities through in-depth web research and competitor analysis.
2). Do competitor analysis for {search_keywords} for content calender & planning.
2). Provide your research to Senior Content Strategist & planner - Ted XingPi
""",
tools = [search_tool],
@@ -75,23 +68,28 @@ def create_agents(search_keywords):
role = 'Senior Content Strategist & planner - Ted XingPi',
goal = f"""
Craft a series of content titles around {search_keywords} that can be expanded into 2 month-long series.
Do not repeat the blog titles, always consult the previously written blog titles from the file(content_already_planned.txt).
Set the """,
Do not repeat the blog titles, always consult the previously written blog titles from the file: {already_written_on}.""",
backstory = """You are Ted XingPi, with Experience of 15 years.
Your Skills:
1). Content Opportunity Analysis & Content calender planning (Expert).
2). AI Applications for Content Marketing (Highly Knowledgeable).
3). Content Strategy Development & planning.
4). Analyze keyword research for content opportunities.
3). Content Strategy Development & keyword research for content opportunities.
Your Responsibilties:
1). Identify content topics and keywords for {search_keywords}.
2). Create content calender that showcases the value proposition around {search_keywords}.
3). Collaborate with team to identify content gaps and trending topics, relevant to given keywords.
4). Develop content calender with a focus on organic marketing to attract online customers.
5). The content calender should include, Head Term Keyword, Long-Tail Keyword and Blog Post Title.
1). Employ a balance of head terms (broad topics) and long-tail keywords (specific phrases) for optimal reach and targeting.
2). Review & Include suggestions from Content Marketing & Google Trends Specialist - Sarah Qureshi.
3). Identify content topics and keywords for {search_keywords}.
4). Senior Web Research Analyst (Content Strategy): Aisha Sharma
5). Create content calender that showcases the value proposition around {search_keywords}.
6). New content should target unique keywords to avoid competition with existing content.
7). Focus on specific aspects within a theme to differentiate semantically similar keywords for {search_keywords}.
8). Collaborate with team to identify content gaps and trending topics, relevant to given keywords.
9). Develop content calender with a focus on organic marketing to attract online customers.
10). The content calender should include, Head Term Keyword, Long-Tail Keyword and Blog Post Title.
""",
memory = True, # Enable memory
verbose = True,
@@ -105,25 +103,21 @@ def create_agents(search_keywords):
google_trends_researcher = Agent(
role = 'Content Marketing & Google Trends Specialist - Sarah Qureshi.',
goal = f"""Use Google Trends to suggest AI writing keywords & titles, optimized for {search_keywords}.
All the required google trends data is present in the file({os.getenv('SEARCH_SAVE_FILE')}).""",
goal = f"""Help Create a highly detailed 2 month-long content calender, focused around keywords: {search_keywords}.
Analyse & provide Google trends data for content calender & planning to Ted XingPi""",
backstory = f"""You are Sarah Qureshi, with 10 years as a content writer and planner.
Your Skills:
1). Proven experience in using Google Trends for keyword research.
2). Strong understanding of SEO best practices.
3). Reading files and understanding long table with data.
4). Ability to communicate complex data insights in a clear and concise way.
Your Personality:
1). Enjoys breaking down complex topics into clear and concise information.
2). Strong communicator with a knack for explaining technical concepts in an engaging way.
Your responsibilties:
1). Collaborate on content strategy, provide keyword, titles recommendations and integrate them into the content calendar.
1). Collaborate on content strategy, provide keyword, titles recommendations to Ted XingPi.
2). Recommend high-volume, low-competition keywords, titles with strong user intent.
3). Recommend, Rising search queries related to {search_keywords}.
5). Recommend keywords, blog titles for preparing/planning the content calender.
4). Recommend keywords, blog titles for preparing/planning the content calender.
5). Provide your research to Senior Content Strategist & planner - Ted XingPi
""",
memory = True, # Enable memory
tools = [file_read_tool],
@@ -137,7 +131,7 @@ def create_agents(search_keywords):
content_marketing_manager = Agent(
role="Content Marketing Manager - Diksha Yuj",
goal=f"""Create highly detailed 2 month-long content calender, focused around keywords: {search_keywords}.
Important to ensure the keywords and titles are not already written on, consult the file(content_already_planned.txt).""",
Use insights and context from team members: Sarah Qureshi, Ted XingPi and Aisha Sharma""",
backstory="""
Content Marketing Manager: Diksha Yuj
Experience: Digital Marketing Veteran (15+ years)
@@ -147,10 +141,11 @@ def create_agents(search_keywords):
Responsibilities:
1). Ensures that content titles are not repeated & No keyword cannabilization.
2). Maintains and consults a file for all previous written titles.
2). Maintains and consults a file for all previous written titles({already_written_on}).
3). Develops a content calendar aligned and optimized around {search_keywords}.
4). Keenly follows & learns the research and communication of other team members.
5). The content calender should include, Head Term Keyword, Long-Tail Keyword and Blog Post Title.
6). Use insights and context from team members: Sarah Qureshi, Ted XingPi and Aisha Sharma
""",
memory=True, # Enable memory
verbose=True,
@@ -164,31 +159,28 @@ def create_agents(search_keywords):
return [content_researcher, google_trends_researcher, content_planner, content_marketing_manager]
def create_tasks(agents, search_keywords):
def create_tasks(agents, search_keywords, already_written_on):
research_task = Task(
description=f"""Conduct a analysis on the following: "{search_keywords}". Suggest blog titles for content calender.
description=f"""Conduct web analysis on "{search_keywords}",for content calender.
Set the input parameter as : search_query""",
expected_output=f"""Analyze keywords {search_keywords} to provide content ideas for content calender.""",
expected_output=f"""Provide comprehensive content calender ideas to Senior Content Strategist & planner - Ted XingPi""",
agent=agents[0] # Assign to the researcher agent
)
google_trends_task = Task(
description=f"""Conduct Google Trends analysis, on keywords: {search_keywords}, from the file({os.getenv('SEARCH_SAVE_FILE')}).
Suggest blog titles for content calender. Recommend high-volume, low-competition keywords with strong user intent.
Set the input parameter file_path to {os.getenv('SEARCH_SAVE_FILE')}""",
expected_output=f"Recommend content ideas, blog titles to be included in the content calender.",
Set the input parameter as : file_path""",
expected_output=f"Provide comprehensive content calender ideas to Senior Content Strategist & planner - Ted XingPi",
agent=agents[1] # Assign to the researcher agent
)
planner_task = Task(
description=f"""Develop a content calendar for {search_keywords} that includes evergreen, trending, and seasonal post ideas.""",
expected_output=f"""
A Highly detailed content calender that positions {search_keywords} as a must-read for industry insiders and newcomers alike.
Prioritize keywords relevant to user needs and search intent, leveraging Google Trends insights.
Employ a balance of head terms (broad topics) and long-tail keywords (specific phrases) for optimal reach and targeting.
description=f"""Develop a content calendar for {search_keywords}, based team member's.
New content should target unique keywords to avoid competition with existing content.
Focus on specific aspects within a theme to differentiate semantically similar keywords for {search_keywords}.
Use context & insights from Aisha Sharma & Sarah Qureshi.
Set the input parameter file_path to content_already_planned.txt""",
Set the input parameter file_path to {already_written_on}""",
expected_output=f"""A Highly detailed content calender that positions {search_keywords} as a must-read for industry insiders and newcomers alike. Final content calender for the next 2 months. Targeting 5 articles per week.
""",
#human_input=True,
agent=agents[2] # Assign to the outliner agent
)
@@ -196,11 +188,12 @@ def create_tasks(agents, search_keywords):
marketing_manager_task = Task(
description=f"""Make sure the content calender is optimised for keywords: '{search_keywords}'.
Make sure the titles are unique, semantically unique and mitigate keyword cannabilization.
Use context & insights from Aisha Sharma & Sarah Qureshi.
Set the input parameter file_path to content_already_planned.txt
Use context & insights from Aisha Sharma, Ted XingPi & Sarah Qureshi.
Set the input parameter file_path to {already_written_on}
""",
expected_output=f"""Final content calender for the next 2 months. Targeting 5 articles per week.
Make sure to present the content calender in tabular format. Include details of how to use the content calender.""",
Make sure to present the content calender in tabular format. Include details of how to use the content calender.
""",
agent=agents[3] # Assign to the reviewer agent
)
@@ -220,11 +213,13 @@ def execute_tasks(agents, tasks):
return result
def ai_agents_writers(search_keywords):
def ai_agents_planner(search_keywords):
already_written_on = os.path.join(os.getcwd(), "lib", "content_planning_calender", "content_already_planned.txt")
do_google_trends_analysis(search_keywords)
#setup_environment()
agents = create_agents(search_keywords)
tasks = create_tasks(agents, search_keywords)
agents = create_agents(search_keywords, already_written_on)
tasks = create_tasks(agents, search_keywords, already_written_on)
result = execute_tasks(agents, tasks)
print("########## Final Output Result ############")
print(result)

12
lib/content_planning_calender/requirements.txt
View File

@@ -1,12 +0,0 @@
crewai[tools]
langchain-google-genai
scikit-learn
scipy
matplotlib
pandas
plotly
lxml_html_clean
tabulate
pytrends
loguru
requests_html

61
lib/utils/alwrity_utils.py
View File

@@ -20,7 +20,7 @@ from lib.ai_writers.ai_financial_writer import write_basic_ta_report
from lib.gpt_providers.text_generation.ai_story_writer import ai_story_generator
from lib.gpt_providers.text_generation.ai_essay_writer import ai_essay_generator
from lib.gpt_providers.text_to_image_generation.main_generate_image_from_prompt import generate_image
from lib.content_planning_calender.content_planning_agents_alwrity_crew import ai_agents_writers
from lib.content_planning_calender.content_planning_agents_alwrity_crew import ai_agents_planner
@@ -191,6 +191,7 @@ def do_web_research():
except Exception as err:
print(f"\n💥🤯 [bold red]ERROR 🤯 : Failed to do web research: {err}\n")
def write_story():
""" Alwrity AI Story Writer """
personas = [
@@ -245,7 +246,59 @@ def write_story():
exit(1)
def content_planning():
def ai_content_team():
""" AI Content team for content planning and writing. """
# Define the options for the radio list
options = [
('1', 'AI Content Ideation & Planning Team'),
('2', 'AI Content Creation Team')
]
# Create the radio list dialog
result = radiolist_dialog(
title='Select Purpose based AI Content Team',
text='Please choose one of the following AI Teams:',
values=options,
ok_text='Select',
cancel_text='Cancel'
).run()
# Check the result and display a message
if result == '1':
content_planning_agents()
elif result == '2':
content_creation_agents()
def content_creation_agents():
"""Agents team to create a content calendar"""
while True:
print("________________________________________________________________")
content_keywords = input_dialog(
title='Enter Main Domain Keywords of your business:',
text='Better keywords will generate better blog titles, content calendar. Play with keywords & combine them into a single calendar:',
).run()
# If the user cancels, exit the loop and the function
if content_keywords is None:
print("User cancelled the input. Exiting the content creation process.")
return
if content_keywords and len(content_keywords.split()) >= 2:
break
else:
message_dialog(
title='Error',
text='🚫 Single keywords are just too vague. Try again.'
).run()
try:
ai_agents_writers(content_keywords)
except Exception as err:
print(f"Failed to generate content calendar: {err}")
def content_planning_agents():
""" Agents team to create a content calender """
while True:
print("________________________________________________________________")
@@ -256,7 +309,7 @@ def content_planning():
# If the user cancels, exit the loop
if content_keywords is None:
break
return
if content_keywords and len(content_keywords.split()) >= 2:
break
else:
@@ -265,7 +318,7 @@ def content_planning():
text='🚫 Single keywords are just too vague. Try again.'
).run()
try:
ai_agents_writers(content_keywords)
ai_agents_planner(content_keywords)
except Exception as err:
print(f"Failed to genrate content calender: {err}")

6
lib/workspace/prompts/long_form_ai_writer.prompts
View File

@@ -19,17 +19,19 @@ content_title: |
2. Ensure the title appeals to the target audience of {target_audience}.
3. Review the provided web research results for {content_keywords}. Ensure your title competes effectively against them.
4. Avoid words like: Unleash, ultimate, uncover, discover, elevate, revolutionizing, unveiling, harnessing, dive, delve into, embrace.
5). Provide no explanations for your response and only respond with only one og your best blog title.
Web research results:
"""{{web_research_result}}"""
content_outline: |
As an expert {content_language} content outliner specializing in SEO-optimized content, create a detailed content outline for the given title based on the provided context.
As an expert {content_language} content outliner specializing in {content_type} content, create a highly detailed content outline for the given title based on the provided context.
Title: {{content_title}}
Instructions:
1. Include most of the topics from the given web research results as context.
2. Ensure the outline appeals to the target audience of {target_audience}.
3. Your response should only include the detailed outline, no explanations.
Web research results:
"""{{web_research_result}}"""
@@ -49,7 +51,7 @@ starting_prompt: |
------------
First, silently review the content outline and title. Consider how to begin writing your content.
First, silently review the content outline and title. Consider how to begin writing your content. Take your time.
Start by writing the very beginning of the outline. You are not expected to finish the entire content now.
Your writing should be detailed, only scratching the surface of the first bullet point of your outline.
Write a minimum of 700 words.