kunthawat/ALwrity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP - Streamlit UI, firecrawl - V0.5

Browse Source
This commit is contained in:
ajaysi
2024-06-12 16:01:46 +05:30
parent ccbaa0e4fa
commit f2aa79264e
12 changed files with 201 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

185
lib/ai_writers/ai_essay_writer.py Normal file
View File

@@ -0,0 +1,185 @@
#####################################################
#
# Alwrity, AI essay writer - Essay_Writing_with_Prompt_Chaining
#
#####################################################
import os
from pathlib import Path
from dotenv import load_dotenv
from google.api_core import retry
import google.generativeai as genai
from pprint import pprint
def generate_with_retry(model, prompt):
"""
Generates content from the model with retry handling for errors.
Parameters:
model (GenerativeModel): The generative model to use for content generation.
prompt (str): The prompt to generate content from.
Returns:
str: The generated content.
"""
try:
# FIXME: Need a progress bar here.
return model.generate_content(prompt, request_options={'retry':retry.Retry()})
except Exception as e:
print(f"Error generating content: {e}")
return ""
def ai_essay_generator(essay_title, selected_essay_type, selected_education_level, selected_num_pages):
"""
Write an Essay using prompt chaining and iterative generation.
Parameters:
persona (str): The persona statement for the author.
story_genre (str): The genre of the story.
characters (str): The characters in the story.
"""
print(f"Starting to write Essay on {essay_title}..")
try:
# Define persona and writing guidelines
guidelines = f'''\
Writing Guidelines
As an expert Essay writer and academic researcher, demostrate your world class essay writing skills.
Follow the below writing guidelines for writing your essay:
1). You specialize in {selected_essay_type} essay writing.
2). Your target audiences include readers from {selected_education_level} level.
3). The title of the essay is {essay_title}.
5). The final essay should of {selected_num_pages} words/pages.
3). Plant the seeds of subplots or potential character arc shifts that can be expanded later.
Remember, your main goal is to write as much as you can. If you get through
the story too fast, that is bad. Expand, never summarize.
'''
# Generate prompts
premise_prompt = f'''\
As an expert essay writer, specilizing in {selected_essay_type} essay writing.
Write an Essay title for given keywords {essay_title}.
The title should appeal to audience level of {selected_education_level}.
'''
outline_prompt = f'''\
As an expert essay writer, specilizing in {selected_essay_type} essay writing.
Your Essay title is:
{{premise}}
Write an outline for the essay.
'''
starting_prompt = f'''\
As an expert essay writer, specilizing in {selected_essay_type} essay writing.
Your essay title is:
{{premise}}
The outline of the Essay is:
{{outline}}
First, silently review the outline and the essay title. Consider how to start the Essay.
Start to write the very beginning of the Essay. You are not expected to finish
the whole Essay now. Your writing should be detailed enough that you are only
scratching the surface of the first bullet of your outline. Try to write AT
MINIMUM 1000 WORDS.
{guidelines}
'''
continuation_prompt = f'''\
As an expert essay writer, specilizing in {selected_essay_type} essay writing.
Your essay title is:
{{premise}}
The outline of the Essay is:
{{outline}}
You've begun to write the essay and continue to do so.
Here's what you've written so far:
{{story_text}}
=====
First, silently review the outline and essay so far.
Identify what the single next part of your outline you should write.
Your task is to continue where you left off and write the next part of the Essay.
You are not expected to finish the whole essay now. Your writing should be
detailed enough that you are only scratching the surface of the next part of
your outline. Try to write AT MINIMUM 1000 WORDS. However, only once the essay
is COMPLETELY finished, write IAMDONE. Remember, do NOT write a whole chapter
right now.
{guidelines}
'''
# Configure generative AI
load_dotenv(Path('../.env'))
genai.configure(api_key=os.getenv('GEMINI_API_KEY'))
# Initialize the generative model
model = genai.GenerativeModel('gemini-pro')
# Generate prompts
try:
premise = generate_with_retry(model, premise_prompt).text
print(f"The title of the Essay is: {premise}")
except Exception as err:
print(f"Essay title Generation Error: {err}")
return
outline = generate_with_retry(model, outline_prompt.format(premise=premise)).text
print(f"The Outline of the essay is: {outline}\n\n")
if not outline:
print("Failed to generate Essay outline. Exiting...")
return
try:
starting_draft = generate_with_retry(model,
starting_prompt.format(premise=premise, outline=outline)).text
pprint(starting_draft)
except Exception as err:
print(f"Failed to Generate Essay draft: {err}")
return
try:
draft = starting_draft
continuation = generate_with_retry(model,
continuation_prompt.format(premise=premise, outline=outline, story_text=draft)).text
pprint(continuation)
except Exception as err:
print(f"Failed to write the initial draft: {err}")
# Add the continuation to the initial draft, keep building the story until we see 'IAMDONE'
try:
draft += '\n\n' + continuation
except Exception as err:
print(f"Failed as: {err} and {continuation}")
while 'IAMDONE' not in continuation:
try:
continuation = generate_with_retry(model,
continuation_prompt.format(premise=premise, outline=outline, story_text=draft)).text
draft += '\n\n' + continuation
except Exception as err:
print(f"Failed to continually write the Essay: {err}")
return
# Remove 'IAMDONE' and print the final story
final = draft.replace('IAMDONE', '').strip()
pprint(final)
except Exception as e:
print(f"Main Essay writing: An error occurred: {e}")

188
lib/ai_writers/ai_story_writer.py Normal file
View File

@@ -0,0 +1,188 @@
#####################################################
#
# google-gemini-cookbook - Story_Writing_with_Prompt_Chaining
#
#####################################################
import os
from pathlib import Path
from dotenv import load_dotenv
from google.api_core import retry
import google.generativeai as genai
from pprint import pprint
def generate_with_retry(model, prompt):
"""
Generates content from the model with retry handling for errors.
Parameters:
model (GenerativeModel): The generative model to use for content generation.
prompt (str): The prompt to generate content from.
Returns:
str: The generated content.
"""
try:
# FIXME: Need a progress bar here.
return model.generate_content(prompt, request_options={'retry':retry.Retry()})
except Exception as e:
print(f"Error generating content: {e}")
return ""
def ai_story_generator(persona, story_genre, characters):
"""
Write a story using prompt chaining and iterative generation.
Parameters:
persona (str): The persona statement for the author.
story_genre (str): The genre of the story.
characters (str): The characters in the story.
"""
print(f"Starting to write {story_genre} story based on characters: {characters}..")
try:
# Define persona and writing guidelines
guidelines = f'''\
Writing Guidelines
Delve deeper. Lose yourself in the world you're building. Unleash vivid
descriptions to paint the scenes in your reader's mind.
Develop your characters—let their motivations, fears, and complexities unfold naturally.
Weave in the threads of your outline, but don't feel constrained by it.
Allow your story to surprise you as you write. Use rich imagery, sensory details, and
evocative language to bring the setting, characters, and events to life.
Introduce elements subtly that can blossom into complex subplots, relationships,
or worldbuilding details later in the story.
Keep things intriguing but not fully resolved.
Avoid boxing the story into a corner too early.
Plant the seeds of subplots or potential character arc shifts that can be expanded later.
Remember, your main goal is to write as much as you can. If you get through
the story too fast, that is bad. Expand, never summarize.
'''
# Generate prompts
premise_prompt = f'''\
{persona}
Write a single sentence premise for a {story_genre} story featuring {characters}.
'''
outline_prompt = f'''\
{persona}
You have a gripping premise in mind:
{{premise}}
Write an outline for the plot of your story.
'''
starting_prompt = f'''\
{persona}
You have a gripping premise in mind:
{{premise}}
Your imagination has crafted a rich narrative outline:
{{outline}}
First, silently review the outline and the premise. Consider how to start the
story.
Start to write the very beginning of the story. You are not expected to finish
the whole story now. Your writing should be detailed enough that you are only
scratching the surface of the first bullet of your outline. Try to write AT
MINIMUM 5000 WORDS.
{guidelines}
'''
continuation_prompt = f'''\
{persona}
You have a gripping premise in mind:
{{premise}}
Your imagination has crafted a rich narrative outline:
{{outline}}
You've begun to immerse yourself in this world, and the words are flowing.
Here's what you've written so far:
{{story_text}}
=====
First, silently review the outline and story so far. Identify what the single
next part of your outline you should write.
Your task is to continue where you left off and write the next part of the story.
You are not expected to finish the whole story now. Your writing should be
detailed enough that you are only scratching the surface of the next part of
your outline. Try to write AT MINIMUM 2000 WORDS. However, only once the story
is COMPLETELY finished, write IAMDONE. Remember, do NOT write a whole chapter
right now.
{guidelines}
'''
# Configure generative AI
load_dotenv(Path('../.env'))
genai.configure(api_key=os.getenv('GEMINI_API_KEY'))
# Initialize the generative model
model = genai.GenerativeModel('gemini-1.0-pro')
# Generate prompts
try:
premise = generate_with_retry(model, premise_prompt).text
print(f"The premise of the story is: {premise}")
except Exception as err:
print(f"Premise Generation Error: {err}")
return
outline = generate_with_retry(model, outline_prompt.format(premise=premise)).text
print(f"The Outline of the story is: {outline}\n\n")
if not outline:
print("Failed to generate outline. Exiting...")
return
# Generate starting draft
try:
starting_draft = generate_with_retry(model,
starting_prompt.format(premise=premise, outline=outline)).text
except Exception as err:
print(f"Failed to Generate Story draft: {err}")
return
try:
draft = starting_draft
continuation = generate_with_retry(model,
continuation_prompt.format(premise=premise, outline=outline, story_text=draft)).text
except Exception as err:
print(f"Failed to write the initial draft: {err}")
# Add the continuation to the initial draft, keep building the story until we see 'IAMDONE'
try:
draft += '\n\n' + continuation
except Exception as err:
print(f"Failed as: {err} and {continuation}")
while 'IAMDONE' not in continuation:
try:
continuation = generate_with_retry(model,
continuation_prompt.format(premise=premise, outline=outline, story_text=draft)).text
draft += '\n\n' + continuation
except Exception as err:
print(f"Failed to continually write the story: {err}")
return
# Remove 'IAMDONE' and print the final story
final = draft.replace('IAMDONE', '').strip()
print(final)
except Exception as e:
print(f"Main Story writing: An error occurred: {e}")

10
lib/ai_writers/keywords_to_blog_streamlit.py
View File

@@ -75,8 +75,6 @@ def write_blog_from_keywords(search_keywords, url=None):
logger.info(f"\n\n######### Blog content after Tavily AI research: ######### \n\n")
blog_markdown_str = write_blog_google_serp(search_keywords, tavily_search_result)
status.update(label="Finished Writing Blog From Tavily Results:{blog_markdown_str}")
else:
print("Not Writing with TAVILY..\n\n")
except Exception as err:
logger.error(f"Failed to do Tavily AI research: {err}")
@@ -96,10 +94,10 @@ def write_blog_from_keywords(search_keywords, url=None):
status.update(label=f"Saved the content in this file: {saved_blog_to_file}")
blog_frontmatter = dedent(f"""
\n---------------------------------------------------------------------
title: {blog_title}\n
categories: [{blog_categories}]\n
tags: [{blog_tags}]\n
Meta description: {blog_meta_desc.replace(":", "-")}\n
title: {blog_title.strip()}\n
categories: [{blog_categories.strip()}]\n
tags: [{blog_tags.strip()}]\n
Meta description: {blog_meta_desc.replace(":", "-").strip()}\n
---------------------------------------------------------------------\n
""")
logger.info(f"\n\n --------- Finished writing Blog for : {search_keywords} -------------- \n")

100
lib/ai_writers/web_url_ai_writer.py
View File

@@ -1,6 +1,8 @@
import sys
import os
from textwrap import dedent
import json
from pathlib import Path
from datetime import datetime
import streamlit as st
@@ -14,14 +16,11 @@ logger.add(sys.stdout,
format="<level>{level}</level>|<green>{file}:{line}:{function}</green>| {message}"
)
from ..ai_web_researcher.gpt_online_researcher import do_google_serp_search,\
do_tavily_ai_search, do_metaphor_ai_research, do_google_pytrends_analysis
from ..ai_web_researcher.firecrawl_web_crawler import scrape_url
from .blog_from_google_serp import write_blog_google_serp, blog_with_research
from ..ai_web_researcher.you_web_reseacher import get_rag_results, search_ydc_index
from ..blog_metadata.get_blog_metadata import blog_metadata
from ..blog_postprocessing.save_blog_to_file import save_blog_to_file
from ..gpt_providers.text_to_image_generation.main_generate_image_from_prompt import generate_image
from ..gpt_providers.text_generation.main_text_generation import llm_text_gen
def blog_from_url(weburl):
@@ -38,62 +37,36 @@ def blog_from_url(weburl):
with st.status("Started Writing..", expanded=True) as status:
st.empty()
status.update(label=f"Researching and Writing Blog on: {weburl}")
scraped_text = scrape_url(weburl)
print(scraped_text)
exit(1)
# Call on the got-researcher, tavily apis for this. Do google search for organic competition.
try:
google_search_result, g_titles = do_google_serp_search(search_keywords)
status.update(label=f"🙎 Finished with Google web for Search: {search_keywords}")
example_blog_titles.append(g_titles)
status.update(label=f"🛀 Starting Tavily AI research: {search_keywords}")
tavily_search_result, t_titles, t_answer = do_tavily_ai_search(search_keywords)
status.update(label=f"🙆 Finished Google Search & Tavily AI Search on: {search_keywords}",
state="complete", expanded=False)
scraped_text = scrape_url(weburl)
logger.info(scraped_text)
except Exception as err:
st.error(f"Failed in web research: {err}")
st.error(f"Failed to scrape web page from url-{weburl} - Error: {err}")
logger.error(f"Failed in web research: {err}")
st.stop()
status.update(label="Successfully Scraped/Fetched url: {weburl}", expanded=False, state="complete")
with st.status("Started Writing blog from google search..", expanded=True) as status:
status.update(label="Researching and Writing Blog on keywords.")
# Call on the got-researcher, tavily apis for this. Do google search for organic competition.
try:
status.update(label=f"🛀 Writing blog from Google Search on: {search_keywords}")
blog_markdown_str = write_blog_google_serp(search_keywords, google_search_result)
st.markdown(blog_markdown_str)
status.update(label="🙎 Draft 1: Your Content from Google search result.", state="complete", expanded=False)
except Exception as err:
st.error(f"Failed in Google web research: {err}")
logger.error(f"Failed in Google web research: {err}")
# logger.info/check the final blog content.
logger.info("######### Draft1: Finished Blog from Google web search: ###########")
with st.status("Started Writing blog from Tavily Web search..", expanded=True) as status:
with st.status(f"Started Writing blog from {weburl}..", expanded=True) as status:
# Do Tavily AI research to augument the above blog.
try:
#example_blog_titles.append(t_titles)
if blog_markdown_str and tavily_search_result:
logger.info(f"\n\n######### Blog content after Tavily AI research: ######### \n\n")
blog_markdown_str = write_blog_google_serp(search_keywords, tavily_search_result)
status.update(label="Finished Writing Blog From Tavily Results:{blog_markdown_str}")
else:
print("Not Writing with TAVILY..\n\n")
blog_markdown_str = write_blog_from_weburl(scraped_text)
status.update(label="Finished Writing Blog From: {weburl}")
except Exception as err:
logger.error(f"Failed to do Tavily AI research: {err}")
logger.error(f"Failed to write blog from: {weburl}")
st.error(f"Failed to write blog from: {weburl}")
st.stop()
status.update(label="🙎 Generating - Title, Meta Description, Tags, Categories for the content.")
try:
status.update(label="🙎 Generating - Title, Meta Description, Tags, Categories for the content.")
blog_title, blog_meta_desc, blog_tags, blog_categories = blog_metadata(blog_markdown_str)
except Exception as err:
st.error(f"Failed to get blog metadata: {err}")
try:
status.update(label="🙎 Generating Image for the new blog.")
generated_image_filepath = generate_image(f"{blog_title} + ' ' + {blog_meta_desc}")
except Exception as err:
st.error(f"Failed in Image generation: {err}")
st.warning(f"Failed in Image generation: {err}")
saved_blog_to_file = save_blog_to_file(blog_markdown_str, blog_title, blog_meta_desc,
blog_tags, blog_categories, generated_image_filepath)
@@ -106,8 +79,45 @@ def blog_from_url(weburl):
Meta description: {blog_meta_desc.replace(":", "-")}\n
---------------------------------------------------------------------\n
""")
logger.info(f"\n\n --------- Finished writing Blog for : {search_keywords} -------------- \n")
logger.info(f"\n\n --------- Finished writing Blog for : {weburl} -------------- \n")
st.markdown(f"{blog_frontmatter}")
st.image(generated_image_filepath)
st.markdown(f"{blog_markdown_str}")
status.update(label=f"Finished, Review & Use your Original Content Below: {saved_blog_to_file}")
def write_blog_from_weburl(scraped_website):
"""Combine the given online research and GPT blog content"""
try:
config_path = Path(os.environ["ALWRITY_CONFIG"])
with open(config_path, 'r', encoding='utf-8') as file:
config = json.load(file)
except Exception as err:
logger.error(f"Error: Failed to read values from config: {err}")
exit(1)
blog_characteristics = config['Blog Content Characteristics']
prompt = f"""
As expert Creative Content writer, I will provide you with scraped website content.
I want you to write a detailed {blog_characteristics['Blog Type']} blog post including 5 FAQs.
Below are the guidelines to follow:
1). You must respond in {blog_characteristics['Blog Language']} language.
2). Tone and Brand Alignment: Adjust your tone, voice, personality for {blog_characteristics['Blog Tone']} audience.
3). Make sure your response content length is of {blog_characteristics['Blog Length']} words.
4). Include FAQs from 'People also Ask' section of provided context 'google search result'.
I want the post to offer unique insights, relatable examples, and a fresh perspective on the topic.
\n\n
Website Content:
'''{scraped_website}'''
"""
logger.info("Generating blog and FAQs from Google web search results.")
try:
response = llm_text_gen(prompt)
return response
except Exception as err:
logger.error(f"Exit: Failed to get response from LLM: {err}")
exit(1)