Merge branch 'alwrity_keyword_research'

.gitignore

@@ -21,6 +21,72 @@ __pycache__
 *.pywpz
 *.pywpzp
 
+lib/workspace/alwrity_web_research/*
+lib/workspace/alwrity_web_research_cache/*
+web_research_report*
+
+.swp
+.swo
+.swn
+.swnw
+.swnwp
+.swnwpz
+.swnwpzp
+
+*.log
+*.log.*
+*.log.*.*
+*.log.*.*.*
+*.log.*.*.*.*
+*.log.*.*.*.*.*
+
+.venv
+*.cpython*
+*.cpython-312.pyc
+
+*venv
+*.venv
+*.venv*
+*.venv_*
+*.venv_*_*
+*.venv_*_*_*
+*.venv_*_*_*_*
+
+*venv
+venv_new*
+venv_*
+
+AI-Writer_cursor_workspace.code-workspace
+*.code-workspace
+.cursorignore
+lib/ai_writers/__pycache__/ai_agents_crew_writer.cpython-312.pyc
+*cpython*
+*.cpython*
+.DS_Store
+.vscode
+*.pyc
+.env
+.env.local
+.env.development.local
+.env.test.local
+pycache
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+*.pyw
+*.pyz
+*.pywz
+*.pyzw
+*.pyzp
+*.pywp
+*.pywpz
+*.pywpzp
+
+lib/workspace/alwrity_web_research/*
+lib/workspace/alwrity_web_research_cache/*
+web_research_report*
+
 .swp
 .swo
 .swn

alwrity.py

@@ -1,11 +1,22 @@
 import streamlit as st
+import os
+import json
+import base64
+import logging
+from datetime import datetime
+<<<<<<< HEAD
+import os
+import json
+import base64
+import logging
+from datetime import datetime
 
 # Set page config - must be the first Streamlit command
 st.set_page_config(
     page_title="AI Writer - Content Generation Platform",
     page_icon="✍️",
     layout="wide",
-    initial_sidebar_state="collapsed",  # Start with collapsed sidebar
+    initial_sidebar_state="expanded",  # Changed from collapsed to expanded
     menu_items={
         'Get Help': None,
         'Report a bug': None,
@@ -13,21 +24,102 @@ st.set_page_config(
     }
 )
 
-# Add CSS to hide sidebar during setup
-st.markdown("""
+# Load and apply custom CSS
+with open('lib/workspace/alwrity_ui_styling.css', 'r') as f:
+    css = f.read()
+    
+st.markdown(f"""
     <style>
-        #MainMenu {visibility: hidden;}
-        footer {visibility: hidden;}
-        .stDeployButton {display:none;}
-        /* Hide sidebar during setup */
-        [data-testid="stSidebar"] {
+        /* Hide Streamlit header elements */
+        header {{
             visibility: hidden !important;
-            width: 0px !important;
-            position: fixed !important;
-        }
+            height: 0px !important;
+        }}
+        
+        /* Hide Deploy button */
+        .stDeployButton {{
+            display: none !important;
+        }}
+        
+        /* Adjust top padding since we removed the header */
+        .main .block-container {{
+            padding-top: 1rem !important;
+        }}
+        
+        {css}
     </style>
 """, unsafe_allow_html=True)
 
+import os
+import json
+import base64
+import logging
+import logging
+from datetime import datetime
+import os
+import json
+import base64
+import logging
+from datetime import datetime
+
+# Configure logging
+logging.basicConfig(
+    level=logging.DEBUG,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(),  # Output to console
+        #logging.FileHandler('alwrity.log')  # Output to file
+    ]
+)
+logger = logging.getLogger(__name__)
+=======
+>>>>>>> b48a3b1 (Google Search Grounded results, Content Calendar Ideator, Competitor Analysis, and Keyword Researcher)
+
+# Set page config - must be the first Streamlit command
+st.set_page_config(
+    page_title="AI Writer - Content Generation Platform",
+    page_icon="✍️",
+    layout="wide",
+    initial_sidebar_state="expanded",  # Changed from collapsed to expanded
+    menu_items={
+        'Get Help': None,
+        'Report a bug': None,
+        'About': None
+    }
+)
+
+# Load and apply custom CSS
+with open('lib/workspace/alwrity_ui_styling.css', 'r') as f:
+    css = f.read()
+    
+st.markdown(f"""
+    <style>
+        /* Hide Streamlit header elements */
+        header {{
+            visibility: hidden !important;
+            height: 0px !important;
+        }}
+        
+        /* Hide Deploy button */
+        .stDeployButton {{
+            display: none !important;
+        }}
+        
+        /* Adjust top padding since we removed the header */
+        .main .block-container {{
+            padding-top: 1rem !important;
+        }}
+        
+        {css}
+    </style>
+""", unsafe_allow_html=True)
+
+import os
+import json
+import base64
+import logging
+import logging
+from datetime import datetime
 import os
 import json
 import base64
@@ -45,37 +137,11 @@ logging.basicConfig(
 )
 logger = logging.getLogger(__name__)
 
-from lib.utils.config_manager import save_config
 from lib.utils.ui_setup import setup_ui
-from lib.utils.alwrity_sidebar import sidebar_configuration
 from lib.utils.api_key_manager.api_key_manager import APIKeyManager, render
 from lib.utils.api_key_manager.validation import check_all_api_keys
 from dotenv import load_dotenv
-from lib.utils.content_generators import ai_writers, content_planning_tools, blog_from_keyword, story_input_section, essay_writer, ai_news_writer, ai_finance_ta_writer, write_ai_prod_desc, do_web_research, competitor_analysis
-from lib.utils.seo_tools import ai_seo_tools
-from lib.utils.ui_setup import setup_ui, setup_tabs
-from lib.utils.alwrity_utils import ai_agents_team, ai_social_writer
-from lib.utils.file_processor import load_image, read_prompts, write_prompts
-from lib.utils.voice_processing import record_voice
-
-def process_folder_for_rag(folder_path):
-    """Placeholder for the process_folder_for_rag function."""
-    logger.info(f"Processing folder for RAG: {folder_path}")
-    st.write(f"This is a placeholder for processing the folder: {folder_path}")
-
-
-def save_config(config):
-    """
-    Saves the provided configuration dictionary to a JSON file specified by the environment variable.
-    """
-    try:
-        logger.debug(f"Saving configuration to {os.getenv('ALWRITY_CONFIG')}")
-        with open(os.getenv("ALWRITY_CONFIG"), "w") as config_file:
-            json.dump(config, config_file, indent=4)
-        logger.info("Configuration saved successfully")
-    except Exception as e:
-        logger.error(f"Error saving configuration: {str(e)}", exc_info=True)
-        st.error(f"An error occurred while saving the configuration: {e}")
+from lib.utils.ui_setup import setup_ui, setup_alwrity_ui
 
 
 def main():
@@ -94,36 +160,110 @@ def main():
     # Check API keys and show setup if needed
     if not check_all_api_keys(api_key_manager):
         logger.info("API keys not verified")
+        # Add CSS to hide sidebar during setup
+        st.markdown("""
+        <style>
+            #MainMenu {visibility: hidden;}
+            footer {visibility: hidden;}
+            .stDeployButton {display:none;}
+            /* Hide sidebar during setup */
+            [data-testid="stSidebar"] {
+                visibility: hidden !important;
+                width: 0px !important;
+                position: fixed !important;
+            }
+        </style>
+        """, unsafe_allow_html=True)
         render(api_key_manager)
         return
     else:
         logger.info("All API keys verified")
-        # Remove the CSS that hides the sidebar
+        # Remove the CSS that hides the sidebar and ensure it's expanded
         st.markdown("""
             <style>
                 #MainMenu {visibility: visible;}
                 footer {visibility: visible;}
                 .stDeployButton {display:block;}
+                
+                /* Sidebar styling */
                 [data-testid="stSidebar"] {
                     visibility: visible !important;
-                    width: 250px !important;
                     position: relative !important;
+                    transition: width 0.3s ease-in-out;
                 }
+                
+                /* Expanded state */
                 [data-testid="stSidebar"][aria-expanded="true"] {
-                    width: 250px !important;
+                    width: 288px !important;
+                    margin-left: 0 !important;
                 }
+                
+                /* Collapsed state */
                 [data-testid="stSidebar"][aria-expanded="false"] {
-                    width: 250px !important;
+                    width: 0 !important;
+                    margin-left: 0 !important;
                 }
+                
+                /* Main content area adjustments */
                 .main .block-container {
-                    padding-left: 2rem;
+                    padding-left: 2rem !important;
+                    padding-right: 2rem !important;
+                    max-width: none;
+                }
+                
+                /* Ensure content reflows when sidebar is collapsed */
+                @media (max-width: 768px) {
+                    .main .block-container {
+                        padding-left: 1rem !important;
+                        padding-right: 1rem !important;
+                    }
+                }
+            </style>
+            <script>
+                // Force sidebar to be expanded initially
+                document.addEventListener('DOMContentLoaded', function() {
+                    const sidebar = document.querySelector('[data-testid="stSidebar"]');
+                    if (sidebar) {
+                        sidebar.setAttribute('aria-expanded', 'true');
+                        sidebar.style.transition = 'width 0.3s ease-in-out';
+                        
+                        // Handle sidebar content
+                        const sidebarContent = sidebar.querySelector('.css-1d391kg');
+                        if (sidebarContent) {
+                            sidebarContent.style.width = sidebar.getAttribute('aria-expanded') === 'true' ? '288px' : '0px';
+                            sidebarContent.style.display = 'block';
+                            sidebarContent.style.transition = 'width 0.3s ease-in-out';
+                        }
+                        
+                        // Add event listener for sidebar toggle
+                        const toggleButton = document.querySelector('button[kind="header"]');
+                        if (toggleButton) {
+                            toggleButton.addEventListener('click', function() {
+                                const isExpanded = sidebar.getAttribute('aria-expanded') === 'true';
+                                if (sidebarContent) {
+                                    sidebarContent.style.width = isExpanded ? '0px' : '288px';
+                                }
+                            });
+                        }
+                    }
+                });
+            </script>
+        """, unsafe_allow_html=True)
+        
+        # Set session state to ensure sidebar stays expanded
+        if 'sidebar_expanded' not in st.session_state:
+            st.session_state.sidebar_expanded = True
+            
+        # Force sidebar state
+        st.sidebar.markdown("""
+            <style>
+                [data-testid="stSidebar"] {
+                    width: 288px !important;
                 }
             </style>
         """, unsafe_allow_html=True)
-        
-        setup_environment_paths()
-        sidebar_configuration()
-        setup_tabs()
+            
+        setup_alwrity_ui()
 
 
 def setup_environment_paths():
@@ -142,52 +282,5 @@ def setup_environment_paths():
         raise
 
 
-# Functions for the main options
-def ai_writers():
-    options = [
-        "AI Blog Writer",
-        "Story Writer",
-        "Essay writer",
-        "Write News reports",
-        "Write Financial TA report",
-        "AI Product Description Writer",
-        "AI Copywriter",
-        "Quit"
-    ]
-    choice = st.selectbox("**👇Select a content creation type:**", options, index=0, format_func=lambda x: f"📝 {x}")
-
-    if choice == "AI Blog Writer":
-        blog_from_keyword()
-    elif choice == "Story Writer":
-        story_input_section()
-    elif choice == "Essay writer":
-        essay_writer()
-    elif choice == "Write News reports":
-        ai_news_writer()
-    elif choice == "Write Financial TA report":
-        ai_finance_ta_writer()
-    elif choice == "AI Product Description Writer":
-        write_ai_prod_desc()
-    elif choice == "Quit":
-        st.subheader("Exiting, Getting Lost. But.... I have nowhere to go 🥹🥹")
-
-
-
-def alwrity_brain():
-    st.title("🧠 Alwrity Brain, Better than yours!")
-    st.write("Choose a folder to write content on. Alwrity will do RAG on these documents. The documents can of any type, pdf, pptx, docs, txt, cs etc. Video files and Audio files are also permitted.")
-
-    folder_path = st.text_input("**Enter folder path:**")
-    if st.button("**Process Folder**"):
-        if folder_path:
-            try:
-                process_folder_for_rag(folder_path)
-                st.success("Folder processed successfully!")
-            except Exception as e:
-                st.error(f"Error processing folder: {e}")
-        else:
-            st.warning("Please enter a valid folder path.")
-
-
 if __name__ == "__main__":
-    main()
+    main()

lib/ai_web_researcher/gemini_grounding_search_streamlit.py

@@ -0,0 +1,155 @@
+import os
+import streamlit as st
+from google import genai
+from google.genai.types import Tool, GenerateContentConfig, GoogleSearch
+
+# Set page config
+st.set_page_config(
+    page_title="Gemini Grounding Search",
+    page_icon="🔍",
+    layout="wide"
+)
+
+# Custom CSS for styling
+st.markdown("""
+<style>
+.container {
+    align-items: center;
+    border-radius: 8px;
+    display: flex;
+    font-family: Google Sans, Roboto, sans-serif;
+    font-size: 14px;
+    line-height: 20px;
+    padding: 8px 12px;
+    background-color: #fafafa;
+    box-shadow: 0 0 0 1px #0000000f;
+    margin-top: 20px;
+}
+.chip {
+    display: inline-block;
+    border: solid 1px;
+    border-radius: 16px;
+    min-width: 14px;
+    padding: 5px 16px;
+    text-align: center;
+    user-select: none;
+    margin: 0 8px;
+    background-color: #ffffff;
+    border-color: #d2d2d2;
+    color: #5e5e5e;
+    text-decoration: none;
+}
+.chip:hover {
+    background-color: #f2f2f2;
+}
+.carousel {
+    overflow: auto;
+    scrollbar-width: none;
+    white-space: nowrap;
+    margin-right: -12px;
+    display: flex;
+    align-items: center;
+}
+.headline {
+    display: flex;
+    margin-right: 4px;
+    align-items: center;
+}
+.gradient-container {
+    position: relative;
+}
+.gradient {
+    position: absolute;
+    transform: translate(3px, -9px);
+    height: 36px;
+    width: 9px;
+    background: linear-gradient(90deg, #fafafa 15%, #fafafa00 100%);
+}
+.result-text {
+    font-size: 16px;
+    line-height: 1.6;
+    color: #202124;
+    margin: 20px 0;
+    white-space: pre-wrap;
+}
+@media (prefers-color-scheme: dark) {
+    .container {
+        background-color: #1f1f1f;
+        box-shadow: 0 0 0 1px #ffffff26;
+    }
+    .headline-label {
+        color: #fff;
+    }
+    .chip {
+        background-color: #2c2c2c;
+        border-color: #3c4043;
+        color: #fff;
+    }
+    .chip:hover {
+        background-color: #353536;
+    }
+    .gradient {
+        background: linear-gradient(90deg, #1f1f1f 15%, #1f1f1f00 100%);
+    }
+    .result-text {
+        color: #e8eaed;
+    }
+}
+</style>
+""", unsafe_allow_html=True)
+
+# Title
+st.title("Gemini Grounding Search")
+
+# Initialize Gemini client
+if 'GEMINI_API_KEY' not in os.environ:
+    api_key = st.text_input("Enter your Gemini API Key:", type="password")
+    if api_key:
+        os.environ['GEMINI_API_KEY'] = api_key
+
+# Search input
+search_query = st.text_input("Enter your search query:", "When is the next total solar eclipse in the United States?")
+
+if st.button("Search"):
+    if 'GEMINI_API_KEY' not in os.environ:
+        st.error("Please enter your Gemini API Key first!")
+    else:
+        try:
+            client = genai.Client(api_key=os.environ['GEMINI_API_KEY'])
+            model_id = "gemini-2.0-flash"
+
+            google_search_tool = Tool(
+                google_search = GoogleSearch()
+            )
+
+            with st.spinner("Searching..."):
+                response = client.models.generate_content(
+                    model=model_id,
+                    contents=search_query,
+                    config=GenerateContentConfig(
+                        tools=[google_search_tool],
+                        response_modalities=["TEXT"],
+                    )
+                )
+
+                # Display search results header
+                st.header("Search Results")
+                
+                # Display the response text
+                if response.candidates[0].content.parts:
+                    st.markdown('<div class="result-text">' + 
+                              response.candidates[0].content.parts[0].text.replace('\n', '<br>') + 
+                              '</div>', 
+                              unsafe_allow_html=True)
+
+                # Display the grounding metadata
+                if hasattr(response.candidates[0], 'grounding_metadata') and \
+                   hasattr(response.candidates[0].grounding_metadata, 'search_entry_point') and \
+                   hasattr(response.candidates[0].grounding_metadata.search_entry_point, 'rendered_content'):
+                    
+                    st.header("Related Searches")
+                    rendered_content = response.candidates[0].grounding_metadata.search_entry_point.rendered_content
+                    st.markdown(rendered_content, unsafe_allow_html=True)
+
+        except Exception as e:
+            st.error(f"An error occurred: {str(e)}") 

lib/ai_web_researcher/google_trends_researcher.py

@@ -105,124 +105,55 @@ def plot_interest_by_region(kw_list):
 
 
 
-def get_related_queries_and_save_csv(keywords, hl='en-US', tz=360, cat=0, timeframe='today 12-m'):
-    """
-    Get related queries for the given search keywords and save the result to a CSV file.
-
-    Args:
-        search_keywords (list): List of search keywords.
-        hl (str): Language parameter, default is 'en-US'.
-        tz (int): Timezone parameter, default is 360.
-        cat (int): Category parameter, default is 0.
-        timeframe (str): Timeframe parameter, default is 'today 12-m'.
-
-    Returns:
-        pd.DataFrame: DataFrame containing related queries.
-    """
-    try:
-        # Build model
-        pytrends = TrendReq(hl=hl, tz=tz)
-        pytrends.build_payload(kw_list=keywords, cat=cat, timeframe=timeframe)
-
-        # Get related queries
-        data = pytrends.related_queries()
-
-        # Extract data from the result
-        top_queries = list(data.values())[0]['top']
-        rising_queries = list(data.values())[0]['rising']
-        top_rising_queries = top_queries + rising_queries
-
-        # Convert lists to DataFrames
-        df_top_queries = pd.DataFrame(top_queries)
-        df_rising_queries = pd.DataFrame(rising_queries)  # Added this line
-
-        # Rename columns to avoid duplicates
-        df_top_queries.columns = ['Top query', 'value']
-        df_rising_queries.columns = ['Rising query', 'value']
-
-        # Save to CSV
-        all_queries_df = pd.concat([df_top_queries, df_rising_queries], axis=1)
-        #all_queries_df.to_csv('related_queries.csv', index=False)
-
-        # Display additional information
-        console = Console()
-        # Display additional information with emojis and bold formatting
-        print("\n📢❗🚨 ")
-        print("\n\033[1m🔝 Top\033[0m: The most popular search queries. Scoring is on a relative scale where a value of 100 is the most commonly searched query, 50 is a query searched half as often, and a value of 0 is a query searched for less than 1% as often as the most popular query.\n")
-        print("\n\033[1m🚀 Rising\033[0m: Queries with the biggest increase in search frequency since the last time period. Results marked 'Breakout' had a tremendous increase, probably because these queries are new and had few (if any) prior searches.\n")
-        # Display the DataFrame using tabulate
-        table = tabulate(all_queries_df, headers='keys', tablefmt='fancy_grid')
-        print(table)
-        # Save the combined table to a file
-        try:
-            save_in_file(table)
-        except Exception as save_results_err:
-            logger.error(f"Failed to save search results: {save_results_err}")
-        return top_rising_queries
-
-    except Exception as e:
-        print(f"get_related_queries_and_save_csv: ERROR: An error occurred: {e}")
-    
-
 def get_related_topics_and_save_csv(search_keywords):
-    """
-    Get related topics for the given search keywords and save the result to a CSV file.
-
-    Args:
-        search_keywords (list): List of search keywords.
-
-    Returns:
-        pd.DataFrame: DataFrame containing related topics.
-    """
+    search_keywords = [f"{search_keywords}"]
     try:
-        # Build model
         pytrends = TrendReq(hl='en-US', tz=360)
+        pytrends.build_payload(kw_list=search_keywords, timeframe='today 12-m')
         
-        # Build payload
-        # FIXME: Remove hardcoding.
-        pytrends.build_payload(search_keywords, cat=0, timeframe='today 12-m')
-
-        # Get related topics
-        try:
-            data = pytrends.related_topics()
-        except Exception as err:
-            logger.error(f"Failed to get pytrends realted topics: {err}")
-            return None
-
-        # Extract data from the result
-        top_topics = list(data.values())[0]['top']
-        rising_topics = list(data.values())[0]['rising']
+        # Get related topics - this returns a dictionary
+        topics_data = pytrends.related_topics()
         
-        # Convert lists to DataFrames
-        df_top_topics = pd.DataFrame(top_topics)
-        df_rising_topics = pd.DataFrame(rising_topics)
-        
-        # FIXME:Exclude specified columns
-        columns_to_exclude = ['hasData', 'value', 'topic_mid', 'link']
-        df_top_topics = df_top_topics.drop(columns=columns_to_exclude, errors='ignore')
-        df_rising_topics = df_rising_topics.drop(columns=columns_to_exclude, errors='ignore')
-
-        # Rename columns to avoid duplicates and provide meaningful names
-        df_top_topics.columns = ['Top- ' + col if col != 'topic_title' else col for col in df_top_topics.columns]
-        df_rising_topics.columns = ['Rising- ' + col if col != 'topic_title' else col for col in df_rising_topics.columns]
-        all_topics_df = pd.concat([df_top_topics, df_rising_topics], axis=1)
-
-        print(f"\n\n 📢❗🚨 Rising and Trending Keywords for {search_keywords}\n")
-        print("\033[1m🔝 Top\033[0m: The most popular search topics.")
-        print("\033[1m🚀 Rising\033[0m: Topics experiencing a significant increase in search frequency since the last time period. Topics marked :pile_of_poop:'Breakout' had a tremendous surge, likely because they are new and had few prior searches.")
-        # Display the DataFrame using tabulate
-        pd.set_option('display.max_rows', all_topics_df.shape[0]+1)
-        print(all_topics_df.head(10))
-        table = tabulate(all_topics_df, headers='keys', tablefmt='fancy_grid')
-        try:
-            save_in_file(table)
-        except Exception as save_results_err:
-            logger.error(f"Failed to save search results: {save_results_err}")
-        return all_topics_df
-
+        # Extract data for the first keyword
+        if topics_data and search_keywords[0] in topics_data:
+            keyword_data = topics_data[search_keywords[0]]
+            
+            # Create two separate dataframes for top and rising
+            top_df = keyword_data.get('top', pd.DataFrame())
+            rising_df = keyword_data.get('rising', pd.DataFrame())
+            
+            return {
+                'top': top_df[['topic_title', 'value']] if not top_df.empty else pd.DataFrame(),
+                'rising': rising_df[['topic_title', 'value']] if not rising_df.empty else pd.DataFrame()
+            }
     except Exception as e:
-        logger.error(f"ERROR: An error occurred in related topics: {e}")
-        return pd.DataFrame()
+        logger.error(f"Error in related topics: {e}")
+        return {'top': pd.DataFrame(), 'rising': pd.DataFrame()}
+
+def get_related_queries_and_save_csv(search_keywords):
+    search_keywords = [f"{search_keywords}"]
+    try:
+        pytrends = TrendReq(hl='en-US', tz=360)
+        pytrends.build_payload(kw_list=search_keywords, timeframe='today 12-m')
+        
+        # Get related queries - this returns a dictionary
+        queries_data = pytrends.related_queries()
+        
+        # Extract data for the first keyword
+        if queries_data and search_keywords[0] in queries_data:
+            keyword_data = queries_data[search_keywords[0]]
+            
+            # Create two separate dataframes for top and rising
+            top_df = keyword_data.get('top', pd.DataFrame())
+            rising_df = keyword_data.get('rising', pd.DataFrame())
+            
+            return {
+                'top': top_df if not top_df.empty else pd.DataFrame(),
+                'rising': rising_df if not rising_df.empty else pd.DataFrame()
+            }
+    except Exception as e:
+        logger.error(f"Error in related queries: {e}")
+        return {'top': pd.DataFrame(), 'rising': pd.DataFrame()}
 
 
 def get_source(url):
@@ -507,22 +438,17 @@ def do_google_trends_analysis(search_term):
             else:
                 all_the_keywords.append(suggestions_df['Keywords'].tolist())
             all_the_keywords = ','.join([', '.join(filter(None, map(str, sublist))) for sublist in all_the_keywords])
+            
             # Generate a random sleep time between 2 and 3 seconds 
             time.sleep(random.uniform(2, 3))
-
-#        
-#        # FIXME: Get result from vision GPT. Fetch and visualize Google Trends data
-#        #trends_data = fetch_google_trends_interest_overtime("llamaindex")
-#
-#        # FIXME: Plot Interest Over time.
-#        result_df = plot_interest_by_region(search_term)
-#        
+        
         # Display additional information
         try:
             result_df = get_related_topics_and_save_csv(search_term)
+            logger.info(f"Related topics:: result_df: {result_df}")
             # Extract 'Top' topic_title
             if result_df:
-                top_topic_title = result_df['topic_title'].values.tolist()
+                top_topic_title = result_df['top']['topic_title'].values.tolist()
                 # Join each sublist into one string separated by comma
                 #top_topic_title = [','.join(filter(None, map(str, sublist))) for sublist in top_topic_title]
                 top_topic_title = ','.join([', '.join(filter(None, map(str, sublist))) for sublist in top_topic_title])
@@ -551,3 +477,24 @@ def do_google_trends_analysis(search_term):
         return(all_the_keywords)
     except Exception as e:
         logger.error(f"Error in Google Trends Analysis: {e}")
+
+
+def get_trending_searches(country='united_states'):
+    """Get trending searches for a specific country."""
+    try:
+        pytrends = TrendReq(hl='en-US', tz=360)
+        trending_searches = pytrends.trending_searches(pn=country)
+        return trending_searches
+    except Exception as e:
+        logger.error(f"Error getting trending searches: {e}")
+        return pd.DataFrame()
+
+def get_realtime_trends(country='US'):
+    """Get realtime trending searches for a specific country."""
+    try:
+        pytrends = TrendReq(hl='en-US', tz=360)
+        realtime_trends = pytrends.realtime_trending_searches(pn=country)
+        return realtime_trends
+    except Exception as e:
+        logger.error(f"Error getting realtime trends: {e}")
+        return pd.DataFrame()

lib/ai_web_researcher/gpt_online_researcher.py

@@ -22,14 +22,27 @@
 
 import os
 import json
+import time
 from pathlib import Path
 import sys
 from datetime import datetime
+import streamlit as st
+import pandas as pd
+import random
+import numpy as np
+
+from lib.alwrity_ui.display_google_serp_results import (
+    process_research_results,
+    process_search_results,
+    display_research_results
+)
+from lib.alwrity_ui.google_trends_ui import display_google_trends_data, process_trends_data
 
 from .tavily_ai_search import get_tavilyai_results
-from .metaphor_basic_neural_web_search import metaphor_find_similar, metaphor_search_articles
+from .metaphor_basic_neural_web_search import metaphor_search_articles, streamlit_display_metaphor_results
 from .google_serp_search import google_search
 from .google_trends_researcher import do_google_trends_analysis
+#from .google_gemini_web_researcher import do_gemini_web_research
 
 from loguru import logger
 # Configure logger
@@ -40,68 +53,689 @@ logger.add(sys.stdout,
            )
 
 
-
-def gpt_web_researcher(search_keywords):
-    """ Keyword based web researcher, basic, neural and Semantic search."""
+def gpt_web_researcher(search_keywords, search_mode, **kwargs):
+    """Keyword based web researcher with progress tracking."""
+    
+    logger.info(f"Starting web research - Keywords: {search_keywords}, Mode: {search_mode}")
+    logger.debug(f"Additional parameters: {kwargs}")
     
     try:
-        google_search_result = do_google_serp_search(search_keywords)
-        tavily_search_result = do_tavily_ai_search(search_keywords)
-        metaphor_search_result = do_metaphor_ai_research(search_keywords)
-        gtrends_search_result = do_google_pytrends_analysis(search_keywords)
-        # get_rag_results(search_query)
-        print(f"\n\nReview the analysis in this file at: {os.environ.get('SEARCH_SAVE_FILE')}\n")
+        # Reset session state variables for this research operation
+        if 'metaphor_results_displayed' in st.session_state:
+            del st.session_state.metaphor_results_displayed
+        
+        # Initialize result container
+        research_results = None
+        
+        # Create status containers
+        status_container = st.empty()
+        progress_bar = st.progress(0)
+        
+        def update_progress(message, progress=None, level="info"):
+            if progress is not None:
+                progress_bar.progress(progress)
+            if level == "error":
+                status_container.error(f"🚫 {message}")
+            elif level == "warning":
+                status_container.warning(f"⚠️ {message}")
+            else:
+                status_container.info(f"🔄 {message}")
+            logger.debug(f"Progress update [{level}]: {message}")
+
+        if search_mode == "google":
+            logger.info("Starting Google research pipeline")
+            
+            try:
+                # First try Google SERP
+                update_progress("Initiating SERP search...", progress=10)
+                serp_results = do_google_serp_search(search_keywords, **kwargs)
+                
+                if serp_results and serp_results.get('organic'):
+                    logger.info("SERP search successful")
+                    update_progress("SERP search completed", progress=40)
+                    research_results = serp_results
+                else:
+                    logger.warning("SERP search returned no results, falling back to Gemini")
+                    update_progress("No SERP results, trying Gemini...", progress=45)
+                    
+                    # Keep it commented. Fallback to Gemini
+                    #try:
+                    #    gemini_results = do_gemini_web_research(search_keywords)
+                    #    if gemini_results:
+                    #        logger.info("Gemini research successful")
+                    #        update_progress("Gemini research completed", progress=80)
+                    #        research_results = {
+                    #            'source': 'gemini',
+                    #            'results': gemini_results
+                    #        }
+                    #except Exception as gemini_err:
+                    #    logger.error(f"Gemini research failed: {gemini_err}")
+                    #    update_progress("Gemini research failed", level="warning")
+                
+                if research_results:
+                    update_progress("Processing final results...", progress=90)
+                    processed_results = process_research_results(research_results)
+                    
+                    if processed_results:
+                        update_progress("Research completed!", progress=100, level="success")
+                        display_research_results(processed_results)
+                        return processed_results
+                    else:
+                        error_msg = "Failed to process research results"
+                        logger.warning(error_msg)
+                        update_progress(error_msg, level="warning")
+                        return None
+                else:
+                    error_msg = "No results from either SERP or Gemini"
+                    logger.warning(error_msg)
+                    update_progress(error_msg, level="warning")
+                    return None
+                    
+            except Exception as search_err:
+                error_msg = f"Research pipeline failed: {str(search_err)}"
+                logger.error(error_msg, exc_info=True)
+                update_progress(error_msg, level="error")
+                raise
+
+        elif search_mode == "ai":
+            logger.info("Starting AI research pipeline")
+            
+            try:
+                # Do Tavily AI Search
+                update_progress("Initiating Tavily AI search...", progress=10)
+                
+                # Extract relevant parameters for Tavily search
+                include_domains = kwargs.pop('include_domains', None)
+                search_depth = kwargs.pop('search_depth', 'advanced')
+                
+                # Pass the parameters to get_tavilyai_results
+                t_results = get_tavilyai_results(
+                    keywords=search_keywords, 
+                    max_results=kwargs.get('num_results', 10),
+                    include_domains=include_domains,
+                    search_depth=search_depth,
+                    **kwargs
+                )
+                
+                # Do Metaphor AI Search
+                update_progress("Initiating Metaphor AI search...", progress=50)
+                metaphor_results, metaphor_titles = do_metaphor_ai_research(search_keywords)
+                
+                if metaphor_results is None:
+                    update_progress("Metaphor AI search failed, continuing with Tavily results only...", level="warning")
+                else:
+                    update_progress("Metaphor AI search completed successfully", progress=75)
+                    # Add debug logging to check the structure of metaphor_results
+                    logger.debug(f"Metaphor results structure: {type(metaphor_results)}")
+                    if isinstance(metaphor_results, dict):
+                        logger.debug(f"Metaphor results keys: {metaphor_results.keys()}")
+                        if 'data' in metaphor_results:
+                            logger.debug(f"Metaphor data keys: {metaphor_results['data'].keys()}")
+                            if 'results' in metaphor_results['data']:
+                                logger.debug(f"Number of results: {len(metaphor_results['data']['results'])}")
+                    
+                    # Display Metaphor results only if not already displayed
+                    if 'metaphor_results_displayed' not in st.session_state:
+                        st.session_state.metaphor_results_displayed = True
+                        # Make sure to pass the correct parameters to streamlit_display_metaphor_results
+                        streamlit_display_metaphor_results(metaphor_results, search_keywords)
+                
+                # Add Google Trends Analysis
+                update_progress("Initiating Google Trends analysis...", progress=80)
+                try:
+                    # Add an informative message about Google Trends
+                    with st.expander("ℹ️ About Google Trends Analysis", expanded=False):
+                        st.markdown("""
+                        **What is Google Trends Analysis?**
+                        
+                        Google Trends Analysis provides insights into how often a particular search-term is entered relative to the total search-volume across various regions of the world, and in various languages.
+                        
+                        **What data will be shown?**
+                        
+                        - **Related Keywords**: Terms that are frequently searched together with your keyword
+                        - **Interest Over Time**: How interest in your keyword has changed over the past 12 months
+                        - **Regional Interest**: Where in the world your keyword is most popular
+                        - **Related Queries**: What people search for before and after searching for your keyword
+                        - **Related Topics**: Topics that are closely related to your keyword
+                        
+                        **How to use this data:**
+                        
+                        - Identify trending topics in your industry
+                        - Understand seasonal patterns in search behavior
+                        - Discover related keywords for content planning
+                        - Target content to specific regions with high interest
+                        """)
+                    
+                    trends_results = do_google_pytrends_analysis(search_keywords)
+                    if trends_results:
+                        update_progress("Google Trends analysis completed successfully", progress=90)
+                        # Store trends results in the research_results
+                        if metaphor_results:
+                            metaphor_results['trends_data'] = trends_results
+                        else:
+                            # If metaphor_results is None, create a new container for results
+                            metaphor_results = {'trends_data': trends_results}
+                        
+                        # Display Google Trends data using the new UI module
+                        display_google_trends_data(trends_results, search_keywords)
+                    else:
+                        update_progress("Google Trends analysis returned no results", level="warning")
+                except Exception as trends_err:
+                    logger.error(f"Google Trends analysis failed: {trends_err}")
+                    update_progress("Google Trends analysis failed", level="warning")
+                    st.error(f"Error in Google Trends analysis: {str(trends_err)}")
+                
+                # Return the combined results
+                update_progress("Research completed!", progress=100, level="success")
+                return metaphor_results or t_results
+                    
+            except Exception as ai_err:
+                error_msg = f"AI research pipeline failed: {str(ai_err)}"
+                logger.error(error_msg, exc_info=True)
+                update_progress(error_msg, level="error")
+                raise
+
+        else:
+            error_msg = f"Unsupported search mode: {search_mode}"
+            logger.error(error_msg)
+            update_progress(error_msg, level="error")
+            raise ValueError(error_msg)
+            
     except Exception as err:
-        logger.error(f"Failed in gpt_web_researcher: {err}")
+        error_msg = f"Failed in gpt_web_researcher: {str(err)}"
+        logger.error(error_msg, exc_info=True)
+        if 'update_progress' in locals():
+            update_progress(error_msg, level="error")
+        raise
 
 
-def do_google_serp_search(search_keywords):
-    """ COmmon function to do google SERP analysis and return results. """
-
-    # FIXME: Add a return filter to either return full json, titles, PAA, relatedsearches etc.
+def do_google_serp_search(search_keywords, status_container, update_progress, **kwargs):
+    """Perform Google SERP analysis with sidebar progress tracking."""
+    
+    logger.info("="*50)
+    logger.info("Starting Google SERP Search")
+    logger.info("="*50)
+    
     try:
-        logger.info(f"Doing Google search for: {search_keywords}\n")
+        # Validate parameters
+        update_progress("Validating search parameters")
+        status_container.info("📝 Validating parameters...")
+        
+        if not search_keywords or not isinstance(search_keywords, str):
+            logger.error(f"Invalid search keywords: {search_keywords}")
+            raise ValueError("Search keywords must be a non-empty string")
+        
+        # Update search initiation
+        update_progress(f"Initiating search for: '{search_keywords}'")
+        status_container.info("🌐 Querying search API...")
+        logger.info(f"Search params: {kwargs}")
+        
+        # Execute search
         g_results = google_search(search_keywords)
+        
         if g_results:
+            # Log success
+            update_progress("Search completed successfully", "success")
+            
+            # Update statistics
+            stats = f"""Found:
+                - {len(g_results.get('organic', []))} organic results
+                - {len(g_results.get('peopleAlsoAsk', []))} related questions
+                - {len(g_results.get('relatedSearches', []))} related searches"""
+            update_progress(stats)
+            
+            # Process results
+            update_progress("Processing search results")
+            status_container.info("⚡ Processing results...")
+            processed_results = process_search_results(g_results)
+            
+            # Extract titles
+            update_progress("Extracting information")
             g_titles = extract_info(g_results, 'titles')
-            return(g_results, g_titles)
+            
+            # Final success
+            update_progress("Analysis completed successfully", "success")
+            status_container.success("✨ Research completed!")
+            
+            # Clear main status after delay
+            time.sleep(1)
+            status_container.empty()
+            
+            return {
+                'results': g_results,
+                'titles': g_titles,
+                'summary': processed_results,
+                'stats': {
+                    'organic_count': len(g_results.get('organic', [])),
+                    'questions_count': len(g_results.get('peopleAlsoAsk', [])),
+                    'related_count': len(g_results.get('relatedSearches', []))
+                }
+            }
+            
+        else:
+            update_progress("No results found", "warning")
+            status_container.warning("⚠️ No results found")
+            return None
+            
     except Exception as err:
-        logger.error(f"Failed to do Google SERP research: {err}")
-        return None
-        # Not failing, as tavily would do same and then GPT-V to search.
+        error_msg = f"Search failed: {str(err)}"
+        update_progress(error_msg, "error")
+        logger.error(error_msg)
+        logger.debug("Stack trace:", exc_info=True)
+        raise
+        
+    finally:
+        logger.info("="*50)
+        logger.info("Google SERP Search function completed")
+        logger.info("="*50)
 
 
-def do_tavily_ai_search(search_keywords, max_results=10):
+def do_tavily_ai_search(search_keywords, max_results=10, **kwargs):
     """ Common function to do Tavily AI web research."""
     try:
-        # FIXME: Include the follow-up questions as blog FAQs.
         logger.info(f"Doing Tavily AI search for: {search_keywords}")
-        t_results = get_tavilyai_results(search_keywords, max_results)
-        t_titles = tavily_extract_information(t_results, 'titles')
-        t_answer = tavily_extract_information(t_results, 'answer')
-        return(t_results, t_titles, t_answer)
+        
+        # Prepare Tavily search parameters
+        tavily_params = {
+            'max_results': max_results,
+            'search_depth': 'advanced' if kwargs.get('search_depth', 3) > 2 else 'basic',
+            'time_range': kwargs.get('time_range', 'year'),
+            'include_domains': kwargs.get('include_domains', [""]) if kwargs.get('include_domains') else [""]
+        }
+        
+        # Pass the parameters to get_tavilyai_results
+        t_results = get_tavilyai_results(
+            keywords=search_keywords,
+            **tavily_params
+        )
+        
+        if t_results:
+            t_titles = tavily_extract_information(t_results, 'titles')
+            t_answer = tavily_extract_information(t_results, 'answer')
+            return(t_results, t_titles, t_answer)
+        else:
+            logger.warning("No results returned from Tavily AI search")
+            return None, None, None
     except Exception as err:
         logger.error(f"Failed to do Tavily AI Search: {err}")
+        return None, None, None
 
 
 def do_metaphor_ai_research(search_keywords):
-    """ """
+    """
+    Perform Metaphor AI research and return results with titles.
+    
+    Args:
+        search_keywords (str): Keywords to search for
+        
+    Returns:
+        tuple: (response_articles, titles) or (None, None) if search fails
+    """
     try:
-        logger.info(f"Start Semantic/Neural web search with Metahpor: {search_keywords}")
+        logger.info(f"Start Semantic/Neural web search with Metaphor: {search_keywords}")
         response_articles = metaphor_search_articles(search_keywords)
-        m_titles = metaphor_extract_titles_or_text(response_articles, return_titles=True)
-        return(response_articles, m_titles)
+        
+        if response_articles and 'data' in response_articles:
+            m_titles = [result.get('title', '') for result in response_articles['data'].get('results', [])]
+            return response_articles, m_titles
+        else:
+            logger.warning("No valid results from Metaphor search")
+            return None, None
+            
     except Exception as err:
         logger.error(f"Failed to do Metaphor search: {err}")
+        return None, None
 
 
-def do_google_pytrends_analysis(search_keywords):
-    """ """
+def do_google_pytrends_analysis(keywords):
+    """
+    Perform Google Trends analysis for the given keywords.
+    
+    Args:
+        keywords (str): The search keywords to analyze
+        
+    Returns:
+        dict: A dictionary containing formatted Google Trends data with the following keys:
+            - related_keywords: List of related keywords
+            - interest_over_time: DataFrame with date and interest columns
+            - regional_interest: DataFrame with country_code, country, and interest columns
+            - related_queries: DataFrame with query and value columns
+            - related_topics: DataFrame with topic and value columns
+    """
+    logger.info(f"Performing Google Trends analysis for keywords: {keywords}")
+    
+    # Create a progress container for Streamlit
+    progress_container = st.empty()
+    progress_bar = st.progress(0)
+    
+    def update_progress(message, progress=None, level="info"):
+        """Helper function to update progress in Streamlit UI"""
+        if progress is not None:
+            progress_bar.progress(progress)
+        
+        if level == "error":
+            progress_container.error(f"🚫 {message}")
+        elif level == "warning":
+            progress_container.warning(f"⚠️ {message}")
+        else:
+            progress_container.info(f"🔄 {message}")
+        logger.debug(f"Progress update [{level}]: {message}")
+    
     try:
-        logger.info(f"Do Google Trends analysis for given keywords: {search_keywords}")
-        return(do_google_trends_analysis(search_keywords))
-    except Exception as err:
-        logger.error(f"Failed to do google trends analysis: {err}")
+        # Initialize the formatted data dictionary
+        formatted_data = {
+            'related_keywords': [],
+            'interest_over_time': pd.DataFrame(),
+            'regional_interest': pd.DataFrame(),
+            'related_queries': pd.DataFrame(),
+            'related_topics': pd.DataFrame()
+        }
+        
+        # Get raw trends data from google_trends_researcher
+        update_progress("Fetching Google Trends data...", progress=10)
+        raw_trends_data = do_google_trends_analysis(keywords)
+        
+        if not raw_trends_data:
+            logger.warning("No Google Trends data returned")
+            update_progress("No Google Trends data returned", level="warning", progress=20)
+            return formatted_data
+        
+        # Process related keywords from the raw data
+        update_progress("Processing related keywords...", progress=30)
+        if isinstance(raw_trends_data, list):
+            formatted_data['related_keywords'] = raw_trends_data
+        elif isinstance(raw_trends_data, dict):
+            if 'keywords' in raw_trends_data:
+                formatted_data['related_keywords'] = raw_trends_data['keywords']
+            if 'interest_over_time' in raw_trends_data:
+                formatted_data['interest_over_time'] = raw_trends_data['interest_over_time']
+            if 'regional_interest' in raw_trends_data:
+                formatted_data['regional_interest'] = raw_trends_data['regional_interest']
+            if 'related_queries' in raw_trends_data:
+                formatted_data['related_queries'] = raw_trends_data['related_queries']
+            if 'related_topics' in raw_trends_data:
+                formatted_data['related_topics'] = raw_trends_data['related_topics']
+        
+        # If we have keywords but missing other data, try to fetch them using pytrends directly
+        if formatted_data['related_keywords'] and (
+            formatted_data['interest_over_time'].empty or 
+            formatted_data['regional_interest'].empty or 
+            formatted_data['related_queries'].empty or 
+            formatted_data['related_topics'].empty
+        ):
+            try:
+                update_progress("Fetching additional data from Google Trends API...", progress=40)
+                from pytrends.request import TrendReq
+                pytrends = TrendReq(hl='en-US', tz=360)
+                
+                # Build payload with the main keyword
+                update_progress("Building search payload...", progress=45)
+                pytrends.build_payload([keywords], timeframe='today 12-m', geo='')
+                
+                # Get interest over time if missing
+                if formatted_data['interest_over_time'].empty:
+                    try:
+                        update_progress("Fetching interest over time data...", progress=50)
+                        interest_df = pytrends.interest_over_time()
+                        if not interest_df.empty:
+                            formatted_data['interest_over_time'] = interest_df.reset_index()
+                            update_progress(f"Successfully fetched interest over time data with {len(formatted_data['interest_over_time'])} data points", progress=55)
+                        else:
+                            update_progress("No interest over time data available", level="warning", progress=55)
+                    except Exception as e:
+                        logger.error(f"Error fetching interest over time: {e}")
+                        update_progress(f"Error fetching interest over time: {str(e)}", level="warning", progress=55)
+                
+                # Get regional interest if missing
+                if formatted_data['regional_interest'].empty:
+                    try:
+                        update_progress("Fetching regional interest data...", progress=60)
+                        regional_df = pytrends.interest_by_region()
+                        if not regional_df.empty:
+                            formatted_data['regional_interest'] = regional_df.reset_index()
+                            update_progress(f"Successfully fetched regional interest data for {len(formatted_data['regional_interest'])} regions", progress=65)
+                        else:
+                            update_progress("No regional interest data available", level="warning", progress=65)
+                    except Exception as e:
+                        logger.error(f"Error fetching regional interest: {e}")
+                        update_progress(f"Error fetching regional interest: {str(e)}", level="warning", progress=65)
+                
+                # Get related queries if missing
+                if formatted_data['related_queries'].empty:
+                    try:
+                        update_progress("Fetching related queries data...", progress=70)
+                        # Get related queries data
+                        related_queries = pytrends.related_queries()
+                        
+                        # Create empty DataFrame as fallback
+                        formatted_data['related_queries'] = pd.DataFrame(columns=['query', 'value'])
+                        
+                        # Simple direct approach to avoid list index errors
+                        if related_queries and isinstance(related_queries, dict):
+                            # Check if our keyword exists in the results
+                            if keywords in related_queries:
+                                keyword_data = related_queries[keywords]
+                                
+                                # Process top queries if available
+                                if 'top' in keyword_data and keyword_data['top'] is not None:
+                                    try:
+                                        update_progress("Processing top related queries...", progress=75)
+                                        # Convert to DataFrame if it's not already
+                                        if isinstance(keyword_data['top'], pd.DataFrame):
+                                            top_df = keyword_data['top']
+                                        else:
+                                            # Try to convert to DataFrame
+                                            top_df = pd.DataFrame(keyword_data['top'])
+                                        
+                                        # Ensure it has the right columns
+                                        if not top_df.empty:
+                                            # Rename columns if needed
+                                            if 'query' in top_df.columns:
+                                                # Already has the right column name
+                                                pass
+                                            elif len(top_df.columns) > 0:
+                                                # Use first column as query
+                                                top_df = top_df.rename(columns={top_df.columns[0]: 'query'})
+                                            
+                                            # Add to our results
+                                            formatted_data['related_queries'] = top_df
+                                            update_progress(f"Successfully processed {len(top_df)} top related queries", progress=80)
+                                    except Exception as e:
+                                        logger.warning(f"Error processing top queries: {e}")
+                                        update_progress(f"Error processing top queries: {str(e)}", level="warning", progress=80)
+                                
+                                # Process rising queries if available
+                                if 'rising' in keyword_data and keyword_data['rising'] is not None:
+                                    try:
+                                        update_progress("Processing rising related queries...", progress=85)
+                                        # Convert to DataFrame if it's not already
+                                        if isinstance(keyword_data['rising'], pd.DataFrame):
+                                            rising_df = keyword_data['rising']
+                                        else:
+                                            # Try to convert to DataFrame
+                                            rising_df = pd.DataFrame(keyword_data['rising'])
+                                        
+                                        # Ensure it has the right columns
+                                        if not rising_df.empty:
+                                            # Rename columns if needed
+                                            if 'query' in rising_df.columns:
+                                                # Already has the right column name
+                                                pass
+                                            elif len(rising_df.columns) > 0:
+                                                # Use first column as query
+                                                rising_df = rising_df.rename(columns={rising_df.columns[0]: 'query'})
+                                            
+                                            # Combine with existing data if we have any
+                                            if not formatted_data['related_queries'].empty:
+                                                formatted_data['related_queries'] = pd.concat([formatted_data['related_queries'], rising_df])
+                                                update_progress(f"Successfully processed {len(rising_df)} rising related queries", progress=90)
+                                            else:
+                                                formatted_data['related_queries'] = rising_df
+                                                update_progress(f"Successfully processed {len(rising_df)} rising related queries", progress=90)
+                                    except Exception as e:
+                                        logger.warning(f"Error processing rising queries: {e}")
+                                        update_progress(f"Error processing rising queries: {str(e)}", level="warning", progress=90)
+                    except Exception as e:
+                        logger.error(f"Error fetching related queries: {e}")
+                        update_progress(f"Error fetching related queries: {str(e)}", level="warning", progress=90)
+                        # Ensure we have an empty DataFrame with the right columns
+                        formatted_data['related_queries'] = pd.DataFrame(columns=['query', 'value'])
+                
+                # Get related topics if missing
+                if formatted_data['related_topics'].empty:
+                    try:
+                        update_progress("Fetching related topics data...", progress=95)
+                        # Get related topics data
+                        related_topics = pytrends.related_topics()
+                        
+                        # Create empty DataFrame as fallback
+                        formatted_data['related_topics'] = pd.DataFrame(columns=['topic', 'value'])
+                        
+                        # Simple direct approach to avoid list index errors
+                        if related_topics and isinstance(related_topics, dict):
+                            # Check if our keyword exists in the results
+                            if keywords in related_topics:
+                                keyword_data = related_topics[keywords]
+                                
+                                # Process top topics if available
+                                if 'top' in keyword_data and keyword_data['top'] is not None:
+                                    try:
+                                        update_progress("Processing top related topics...", progress=97)
+                                        # Convert to DataFrame if it's not already
+                                        if isinstance(keyword_data['top'], pd.DataFrame):
+                                            top_df = keyword_data['top']
+                                        else:
+                                            # Try to convert to DataFrame
+                                            top_df = pd.DataFrame(keyword_data['top'])
+                                        
+                                        # Ensure it has the right columns
+                                        if not top_df.empty:
+                                            # Rename columns if needed
+                                            if 'topic_title' in top_df.columns:
+                                                top_df = top_df.rename(columns={'topic_title': 'topic'})
+                                            elif len(top_df.columns) > 0 and 'topic' not in top_df.columns:
+                                                # Use first column as topic
+                                                top_df = top_df.rename(columns={top_df.columns[0]: 'topic'})
+                                            
+                                            # Add to our results
+                                            formatted_data['related_topics'] = top_df
+                                            update_progress(f"Successfully processed {len(top_df)} top related topics", progress=98)
+                                    except Exception as e:
+                                        logger.warning(f"Error processing top topics: {e}")
+                                        update_progress(f"Error processing top topics: {str(e)}", level="warning", progress=98)
+                                
+                                # Process rising topics if available
+                                if 'rising' in keyword_data and keyword_data['rising'] is not None:
+                                    try:
+                                        update_progress("Processing rising related topics...", progress=99)
+                                        # Convert to DataFrame if it's not already
+                                        if isinstance(keyword_data['rising'], pd.DataFrame):
+                                            rising_df = keyword_data['rising']
+                                        else:
+                                            # Try to convert to DataFrame
+                                            rising_df = pd.DataFrame(keyword_data['rising'])
+                                        
+                                        # Ensure it has the right columns
+                                        if not rising_df.empty:
+                                            # Rename columns if needed
+                                            if 'topic_title' in rising_df.columns:
+                                                rising_df = rising_df.rename(columns={'topic_title': 'topic'})
+                                            elif len(rising_df.columns) > 0 and 'topic' not in rising_df.columns:
+                                                # Use first column as topic
+                                                rising_df = rising_df.rename(columns={rising_df.columns[0]: 'topic'})
+                                            
+                                            # Combine with existing data if we have any
+                                            if not formatted_data['related_topics'].empty:
+                                                formatted_data['related_topics'] = pd.concat([formatted_data['related_topics'], rising_df])
+                                                update_progress(f"Successfully processed {len(rising_df)} rising related topics", progress=100)
+                                            else:
+                                                formatted_data['related_topics'] = rising_df
+                                                update_progress(f"Successfully processed {len(rising_df)} rising related topics", progress=100)
+                                    except Exception as e:
+                                        logger.warning(f"Error processing rising topics: {e}")
+                                        update_progress(f"Error processing rising topics: {str(e)}", level="warning", progress=100)
+                    except Exception as e:
+                        logger.error(f"Error fetching related topics: {e}")
+                        update_progress(f"Error fetching related topics: {str(e)}", level="warning", progress=100)
+                        # Ensure we have an empty DataFrame with the right columns
+                        formatted_data['related_topics'] = pd.DataFrame(columns=['topic', 'value'])
+                
+            except Exception as e:
+                logger.error(f"Error fetching additional trends data: {e}")
+                update_progress(f"Error fetching additional trends data: {str(e)}", level="warning", progress=100)
+        
+        # Ensure all DataFrames have the correct column names for the UI
+        update_progress("Finalizing data formatting...", progress=100)
+        
+        if not formatted_data['interest_over_time'].empty:
+            if 'date' not in formatted_data['interest_over_time'].columns:
+                formatted_data['interest_over_time'] = formatted_data['interest_over_time'].reset_index()
+            if 'interest' not in formatted_data['interest_over_time'].columns and keywords in formatted_data['interest_over_time'].columns:
+                formatted_data['interest_over_time'] = formatted_data['interest_over_time'].rename(columns={keywords: 'interest'})
+        
+        if not formatted_data['regional_interest'].empty:
+            if 'country_code' not in formatted_data['regional_interest'].columns and 'geoName' in formatted_data['regional_interest'].columns:
+                formatted_data['regional_interest'] = formatted_data['regional_interest'].rename(columns={'geoName': 'country_code'})
+            if 'interest' not in formatted_data['regional_interest'].columns and keywords in formatted_data['regional_interest'].columns:
+                formatted_data['regional_interest'] = formatted_data['regional_interest'].rename(columns={keywords: 'interest'})
+        
+        if not formatted_data['related_queries'].empty:
+            # Handle different column names that might be present in the related queries DataFrame
+            if 'query' not in formatted_data['related_queries'].columns:
+                if 'Top query' in formatted_data['related_queries'].columns:
+                    formatted_data['related_queries'] = formatted_data['related_queries'].rename(columns={'Top query': 'query'})
+                elif 'Rising query' in formatted_data['related_queries'].columns:
+                    formatted_data['related_queries'] = formatted_data['related_queries'].rename(columns={'Rising query': 'query'})
+                elif 'query' not in formatted_data['related_queries'].columns and len(formatted_data['related_queries'].columns) > 0:
+                    # If we have a DataFrame but no 'query' column, use the first column as 'query'
+                    first_col = formatted_data['related_queries'].columns[0]
+                    formatted_data['related_queries'] = formatted_data['related_queries'].rename(columns={first_col: 'query'})
+            
+            if 'value' not in formatted_data['related_queries'].columns and len(formatted_data['related_queries'].columns) > 1:
+                # If we have a second column, use it as 'value'
+                second_col = formatted_data['related_queries'].columns[1]
+                formatted_data['related_queries'] = formatted_data['related_queries'].rename(columns={second_col: 'value'})
+            elif 'value' not in formatted_data['related_queries'].columns:
+                # If no 'value' column exists, add one with default values
+                formatted_data['related_queries']['value'] = 0
+        
+        if not formatted_data['related_topics'].empty:
+            # Handle different column names that might be present in the related topics DataFrame
+            if 'topic' not in formatted_data['related_topics'].columns:
+                if 'topic_title' in formatted_data['related_topics'].columns:
+                    formatted_data['related_topics'] = formatted_data['related_topics'].rename(columns={'topic_title': 'topic'})
+                elif 'topic' not in formatted_data['related_topics'].columns and len(formatted_data['related_topics'].columns) > 0:
+                    # If we have a DataFrame but no 'topic' column, use the first column as 'topic'
+                    first_col = formatted_data['related_topics'].columns[0]
+                    formatted_data['related_topics'] = formatted_data['related_topics'].rename(columns={first_col: 'topic'})
+            
+            if 'value' not in formatted_data['related_topics'].columns and len(formatted_data['related_topics'].columns) > 1:
+                # If we have a second column, use it as 'value'
+                second_col = formatted_data['related_topics'].columns[1]
+                formatted_data['related_topics'] = formatted_data['related_topics'].rename(columns={second_col: 'value'})
+            elif 'value' not in formatted_data['related_topics'].columns:
+                # If no 'value' column exists, add one with default values
+                formatted_data['related_topics']['value'] = 0
+        
+        # Clear the progress container after completion
+        progress_container.empty()
+        progress_bar.empty()
+        
+        return formatted_data
+        
+    except Exception as e:
+        logger.error(f"Error in Google Trends analysis: {e}")
+        update_progress(f"Error in Google Trends analysis: {str(e)}", level="error", progress=100)
+        # Clear the progress container after error
+        progress_container.empty()
+        progress_bar.empty()
+        return {
+            'related_keywords': [],
+            'interest_over_time': pd.DataFrame(),
+            'regional_interest': pd.DataFrame(),
+            'related_queries': pd.DataFrame(),
+            'related_topics': pd.DataFrame()
+        }
 
 
 def metaphor_extract_titles_or_text(json_data, return_titles=True):
@@ -163,4 +797,4 @@ def tavily_extract_information(json_data, keyword):
     elif keyword == 'follow-query':
         return json_data['follow_up_questions']
     else:
-        return f"Invalid keyword: {keyword}"
+        return f"Invalid keyword: {keyword}"

lib/ai_web_researcher/metaphor_basic_neural_web_search.py

@@ -116,55 +116,331 @@ def metaphor_find_similar(similar_url):
     return search_response
 
 
-
-def metaphor_search_articles(query):
+def calculate_date_range(time_range: str) -> tuple:
     """
-    Search for articles using the Metaphor API.
+    Calculate start and end dates based on time range selection.
+    
+    Args:
+        time_range (str): One of 'past_day', 'past_week', 'past_month', 'past_year', 'anytime'
+        
+    Returns:
+        tuple: (start_date, end_date) in ISO format with milliseconds
+    """
+    now = datetime.utcnow()
+    end_date = now.strftime('%Y-%m-%dT%H:%M:%S.999Z')
+    
+    if time_range == 'past_day':
+        start_date = (now - timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%S.000Z')
+    elif time_range == 'past_week':
+        start_date = (now - timedelta(weeks=1)).strftime('%Y-%m-%dT%H:%M:%S.000Z')
+    elif time_range == 'past_month':
+        start_date = (now - timedelta(days=30)).strftime('%Y-%m-%dT%H:%M:%S.000Z')
+    elif time_range == 'past_year':
+        start_date = (now - timedelta(days=365)).strftime('%Y-%m-%dT%H:%M:%S.000Z')
+    else:  # anytime
+        start_date = None
+        end_date = None
+    
+    return start_date, end_date
+
+def metaphor_search_articles(query, search_options: dict = None):
+    """
+    Search for articles using the Metaphor/Exa API.
 
     Args:
         query (str): The search query.
-        num_results (int): Number of results to retrieve.
-        use_autoprompt (bool): Whether to use autoprompt.
-        include_domains (list): List of domains to include.
-        time_range (str): Time range for published articles ("day", "week", "month", "year", "anytime").
+        search_options (dict): Search configuration options including:
+            - num_results (int): Number of results to retrieve
+            - use_autoprompt (bool): Whether to use autoprompt
+            - include_domains (list): List of domains to include
+            - time_range (str): One of 'past_day', 'past_week', 'past_month', 'past_year', 'anytime'
+            - exclude_domains (list): List of domains to exclude
 
     Returns:
-        MetaphorResponse: The response from the Metaphor API.
+        dict: Search results and metadata
     """
-    metaphor = get_metaphor_client()
+    exa = get_metaphor_client()
     try:
-        include_domains, start_published_date, num_results, similar_url = cfg_search_param('exa')
-        
-        logger.info(f"Metaphor web search with Date: {start_published_date} and Query: {query}")
+        # Initialize default search options
+        if search_options is None:
+            search_options = {}
+
+        # Get config parameters or use defaults
         try:
-            search_response = metaphor.search_and_contents(
-                query,
-                include_domains=include_domains,
-                use_autoprompt=True,
-                start_published_date=start_published_date,
-                num_results=num_results
-            )
-        except Exception as err:
-            logger.error(f"Failed in metaphor.search_and_contents: {err}")
-        
-        # From each webpage, get a summary of the web page.
-        contents_response = search_response.results
-        # FIXME: Need to summarize for smaller input context window.
-#        for content in tqdm(contents_response, desc="Reading Web URL content:", unit="content"):
-#            summarized_content = summarize_web_content(content.text, "gemini")
-#            content.text = summarized_content
-        
-        print_search_result(contents_response)
+            include_domains, _, num_results, _ = cfg_search_param('exa')
+        except Exception as cfg_err:
+            logger.warning(f"Failed to load config parameters: {cfg_err}. Using defaults.")
+            include_domains = None
+            num_results = 10
+
+        # Calculate date range based on time_range option
+        time_range = search_options.get('time_range', 'anytime')
+        start_published_date, end_published_date = calculate_date_range(time_range)
+
+        # Prepare search parameters
+        search_params = {
+            'num_results': search_options.get('num_results', num_results),
+            'summary': True,  # Always get summaries
+            'include_domains': search_options.get('include_domains', include_domains),
+            'use_autoprompt': search_options.get('use_autoprompt', True),
+        }
+
+        # Add date parameters only if they are not None
+        if start_published_date:
+            search_params['start_published_date'] = start_published_date
+        if end_published_date:
+            search_params['end_published_date'] = end_published_date
+
+        logger.info(f"Exa web search with params: {search_params} and Query: {query}")
+        
+        # Execute search
+        search_response = exa.search_and_contents(
+            query,
+            **search_params
+        )
+            
+        if not search_response or not hasattr(search_response, 'results'):
+            logger.warning("No results returned from Exa search")
+            return None
+
+        # Get cost information safely
+        try:
+            cost_dollars = {
+                'total': float(search_response.cost_dollars['total']),
+            } if hasattr(search_response, 'cost_dollars') else None
+        except Exception as cost_err:
+            logger.warning(f"Error processing cost information: {cost_err}")
+            cost_dollars = None
+
+        # Format response to match expected structure
+        formatted_response = {
+            "data": {
+                "requestId": getattr(search_response, 'request_id', None),
+                "resolvedSearchType": "neural",
+                "results": [
+                    {
+                        "id": result.url,
+                        "title": result.title,
+                        "url": result.url,
+                        "publishedDate": result.published_date if hasattr(result, 'published_date') else None,
+                        "author": getattr(result, 'author', None),
+                        "score": getattr(result, 'score', 0),
+                        "summary": result.summary if hasattr(result, 'summary') else None,
+                        "text": result.text if hasattr(result, 'text') else None,
+                        "image": getattr(result, 'image', None),
+                        "favicon": getattr(result, 'favicon', None)
+                    }
+                    for result in search_response.results
+                ],
+                "costDollars": cost_dollars
+            }
+        }
+
+        # Get AI-generated answer from Metaphor
+        try:
+            exa_answer = get_exa_answer(query)
+            if exa_answer:
+                formatted_response.update(exa_answer)
+        except Exception as exa_err:
+            logger.warning(f"Error getting Exa answer: {exa_err}")
+        
+        # Get AI-generated answer from Tavily
+        try:
+            # Import the function directly from the module
+            import importlib
+            tavily_module = importlib.import_module('lib.ai_web_researcher.tavily_ai_search')
+            if hasattr(tavily_module, 'do_tavily_ai_search'):
+                tavily_response = tavily_module.do_tavily_ai_search(query)
+                if tavily_response and 'answer' in tavily_response:
+                    formatted_response.update({
+                        "tavily_answer": tavily_response.get("answer"),
+                        "tavily_citations": tavily_response.get("citations", []),
+                        "tavily_cost_dollars": tavily_response.get("costDollars", {"total": 0})
+                    })
+            else:
+                logger.warning("do_tavily_ai_search function not found in tavily_ai_search module")
+        except Exception as tavily_err:
+            logger.warning(f"Error getting Tavily answer: {tavily_err}")
+        
+        # Return the formatted response without displaying it
+        # The display will be handled by gpt_web_researcher
+        return formatted_response
 
-        if similar_url:
-            logger.info(f"Doing similar/semantic search for URL: {similar_url}")
-            metaphor_find_similar(similar_url)
-        return contents_response
-    
     except Exception as e:
-        logger.error(f"Error in Metaphor searching articles: {e}")
-        raise
+        logger.error(f"Error in Exa searching articles: {e}")
+        return None
 
+def streamlit_display_metaphor_results(metaphor_response, search_keywords=None):
+    """Display Metaphor search results in Streamlit."""
+    
+    if not metaphor_response:
+        st.error("No search results found.")
+        return
+    
+    # Add debug logging
+    logger.debug(f"Displaying Metaphor results. Type: {type(metaphor_response)}")
+    if isinstance(metaphor_response, dict):
+        logger.debug(f"Metaphor response keys: {metaphor_response.keys()}")
+    
+    # Initialize session state variables if they don't exist
+    if 'search_insights' not in st.session_state:
+        st.session_state.search_insights = None
+    if 'metaphor_response' not in st.session_state:
+        st.session_state.metaphor_response = None
+    if 'insights_generated' not in st.session_state:
+        st.session_state.insights_generated = False
+    
+    # Store the current response in session state
+    st.session_state.metaphor_response = metaphor_response
+    
+    # Display search results
+    st.subheader("🔍 Search Results")
+    
+    # Calculate metrics - handle different data structures
+    results = []
+    if isinstance(metaphor_response, dict):
+        if 'data' in metaphor_response and 'results' in metaphor_response['data']:
+            results = metaphor_response['data']['results']
+        elif 'results' in metaphor_response:
+            results = metaphor_response['results']
+    
+    total_results = len(results)
+    avg_relevance = sum(r.get('score', 0) for r in results) / total_results if total_results > 0 else 0
+    
+    # Display metrics
+    col1, col2 = st.columns(2)
+    with col1:
+        st.metric("Total Results", total_results)
+    with col2:
+        st.metric("Average Relevance Score", f"{avg_relevance:.2f}")
+    
+    # Display AI-generated answers if available
+    if 'tavily_answer' in metaphor_response or 'metaphor_answer' in metaphor_response:
+        st.subheader("🤖 AI-Generated Answers")
+        
+        if 'tavily_answer' in metaphor_response:
+            st.markdown("**Tavily AI Answer:**")
+            st.write(metaphor_response['tavily_answer'])
+        
+        if 'metaphor_answer' in metaphor_response:
+            st.markdown("**Metaphor AI Answer:**")
+            st.write(metaphor_response['metaphor_answer'])
+    
+    # Get Search Insights button
+    if st.button("Generate Search Insights", key="metaphor_generate_insights_button"):
+        st.session_state.insights_generated = True
+        st.rerun()
+    
+    # Display insights if they exist in session state
+    if st.session_state.search_insights:
+        st.subheader("🔍 Search Insights")
+        st.write(st.session_state.search_insights)
+    
+    # Display search results in a data editor
+    st.subheader("📊 Detailed Results")
+    
+    # Prepare data for display
+    results_data = []
+    for result in results:
+        result_data = {
+            'Title': result.get('title', ''),
+            'URL': result.get('url', ''),
+            'Snippet': result.get('summary', ''),
+            'Relevance Score': result.get('score', 0),
+            'Published Date': result.get('publishedDate', '')
+        }
+        results_data.append(result_data)
+    
+    # Create DataFrame
+    df = pd.DataFrame(results_data)
+    
+    # Display the DataFrame if it's not empty
+    if not df.empty:
+        # Configure columns
+        st.dataframe(
+            df,
+            column_config={
+                "Title": st.column_config.TextColumn(
+                    "Title",
+                    help="Title of the search result",
+                    width="large",
+                ),
+                "URL": st.column_config.LinkColumn(
+                    "URL",
+                    help="Link to the search result",
+                    width="medium",
+                    display_text="Visit Article",
+                ),
+                "Snippet": st.column_config.TextColumn(
+                    "Snippet",
+                    help="Summary of the search result",
+                    width="large",
+                ),
+                "Relevance Score": st.column_config.NumberColumn(
+                    "Relevance Score",
+                    help="Relevance score of the search result",
+                    format="%.2f",
+                    width="small",
+                ),
+                "Published Date": st.column_config.DateColumn(
+                    "Published Date",
+                    help="Publication date of the search result",
+                    width="medium",
+                ),
+            },
+            hide_index=True,
+        )
+        
+        # Add popover for snippets
+        st.markdown("""
+        <style>
+        .snippet-popover {
+            position: relative;
+            display: inline-block;
+        }
+        .snippet-popover .snippet-content {
+            visibility: hidden;
+            width: 300px;
+            background-color: #f9f9f9;
+            color: #333;
+            text-align: left;
+            border-radius: 6px;
+            padding: 10px;
+            position: absolute;
+            z-index: 1;
+            bottom: 125%;
+            left: 50%;
+            margin-left: -150px;
+            opacity: 0;
+            transition: opacity 0.3s;
+            box-shadow: 0 2px 5px rgba(0,0,0,0.2);
+        }
+        .snippet-popover:hover .snippet-content {
+            visibility: visible;
+            opacity: 1;
+        }
+        </style>
+        """, unsafe_allow_html=True)
+        
+        # Display snippets with popover
+        st.subheader("📝 Snippets")
+        for i, result in enumerate(results):
+            snippet = result.get('summary', '')
+            if snippet:
+                st.markdown(f"""
+                <div class="snippet-popover">
+                    <strong>{result.get('title', '')}</strong>
+                    <div class="snippet-content">
+                        {snippet}
+                    </div>
+                </div>
+                """, unsafe_allow_html=True)
+    else:
+        st.info("No detailed results available.")
+    
+    # Add a collapsible section for the raw JSON data
+    with st.expander("Research Results (JSON)", expanded=False):
+        st.json(metaphor_response)
 
 
 def metaphor_news_summarizer(news_keywords):
@@ -240,3 +516,56 @@ def metaphor_scholar_search(query, include_domains=None, time_range="anytime"):
         return response
     except Exception as e:
         logger.error(f"Error in searching papers: {e}")
+
+def get_exa_answer(query: str, system_prompt: str = None) -> dict:
+    """
+    Get an AI-generated answer for a query using Exa's answer endpoint.
+    
+    Args:
+        query (str): The search query to get an answer for
+        system_prompt (str, optional): Custom system prompt for the LLM. If None, uses default prompt.
+        
+    Returns:
+        dict: Response containing answer, citations, and cost information
+            {
+                "answer": str,
+                "citations": list[dict],
+                "costDollars": dict
+            }
+    """
+    exa = get_metaphor_client()
+    try:
+        # Use default system prompt if none provided
+        if system_prompt is None:
+            system_prompt = (
+                "I am doing research to write factual content. "
+                "Help me find answers for content generation task. "
+                "Provide detailed, well-structured answers with clear citations."
+            )
+
+        logger.info(f"Getting Exa answer for query: {query}")
+        logger.debug(f"Using system prompt: {system_prompt}")
+
+        # Make API call to get answer with system_prompt parameter
+        result = exa.answer(
+            query,
+            model="exa",
+            text=True  # Include full text in citations
+        )
+
+        if not result or not result.get('answer'):
+            logger.warning("No answer received from Exa")
+            return None
+
+        # Format response to match expected structure
+        response = {
+            "answer": result.get('answer'),
+            "citations": result.get('citations', []),
+            "costDollars": result.get('costDollars', {"total": 0})
+        }
+
+        return response
+
+    except Exception as e:
+        logger.error(f"Error getting Exa answer: {e}")
+        return None

lib/ai_web_researcher/tavily_ai_search.py

@@ -49,17 +49,9 @@ from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 
 @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
-def get_tavilyai_results(keywords, max_results=5):
+def get_tavilyai_results(keywords, max_results=5, include_domains=None, search_depth="advanced", **kwargs):
     """
     Get Tavily AI search results based on specified keywords and options.
-
-    Args:
-        keywords (str): Keywords for Tavily AI search.
-        include_urls (str): Comma-separated URLs to include in the search.
-        search_depth (str, optional): Search depth option (default is "advanced").
-
-    Returns:
-        dict: Tavily AI search results.
     """
     # Run Tavily search
     logger.info(f"Running Tavily search on: {keywords}")
@@ -74,56 +66,100 @@ def get_tavilyai_results(keywords, max_results=5):
         client = TavilyClient(api_key=api_key)
     except Exception as err:
         logger.error(f"Failed to create Tavily client. Check TAVILY_API_KEY: {err}")
-    
-    # Read search config params from the file.
-    try:
-        include_urls = cfg_search_param('tavily')
-    except Exception as err:
-        logger.error(f"Failed to read search params from main_config: {err}")
+        raise
 
     try:
-        if include_urls:
-            tavily_search_result = client.search(keywords, 
-                    search_depth="advanced", 
-                    include_answer=True,
-                    max_results=max_results,
-                    include_domains=include_urls)
-        else:
-            tavily_search_result = client.search(keywords, 
-                    search_depth = "advanced", 
-                    include_answer=True,
-                    max_results=max_results)
+        # Create search parameters exactly matching Tavily's API format
+        tavily_search_result = client.search(
+            query=keywords,
+            search_depth="advanced",
+            time_range="year",
+            include_answer="advanced",
+            include_domains=[""] if not include_domains else include_domains,
+            max_results=max_results
+        )
+
+        if tavily_search_result:
+            print_result_table(tavily_search_result)
+            streamlit_display_results(tavily_search_result)
+            return tavily_search_result
+        return None
 
-        print_result_table(tavily_search_result)
-        streamlit_display_results(tavily_search_result)
-        return(tavily_search_result)
     except Exception as err:
         logger.error(f"Failed to do Tavily Research: {err}")
+        raise
 
 
 def streamlit_display_results(output_data):
-    """Display Tavily AI search results in Streamlit UI."""
+    """Display Tavily AI search results in Streamlit UI with enhanced visualization."""
 
-    # Prepare data for display
-    table_data = []
+    # Display the 'answer' in Streamlit with enhanced styling
+    answer = output_data.get("answer", "No answer available")
+    st.markdown("### 🤖 AI-Generated Answer")
+    st.markdown(f"""
+    <div style="background-color: #f0f2f6; padding: 20px; border-radius: 10px; border-left: 5px solid #4CAF50;">
+        {answer}
+    </div>
+    """, unsafe_allow_html=True)
+    
+    # Display follow-up questions if available
+    follow_up_questions = output_data.get("follow_up_questions", [])
+    if follow_up_questions:
+        st.markdown("### ❓ Follow-up Questions")
+        for i, question in enumerate(follow_up_questions, 1):
+            st.markdown(f"**{i}.** {question}")
+    
+    # Prepare data for display with dataeditor
+    st.markdown("### 📊 Search Results")
+    
+    # Create a DataFrame for the results
+    import pandas as pd
+    results_data = []
+    
     for item in output_data.get("results", []):
         title = item.get("title", "")
         snippet = item.get("content", "")
         link = item.get("url", "")
-        table_data.append([title, snippet, link])
+        results_data.append({
+            "Title": title,
+            "Content": snippet,
+            "Link": link
+        })
+    
+    if results_data:
+        df = pd.DataFrame(results_data)
+        
+        # Display the data editor
+        st.data_editor(
+            df,
+            column_config={
+                "Title": st.column_config.TextColumn(
+                    "Title",
+                    help="Article title",
+                    width="medium",
+                ),
+                "Content": st.column_config.TextColumn(
+                    "Content",
+                    help="Click the button below to view full content",
+                    width="large",
+                ),
+                "Link": st.column_config.LinkColumn(
+                    "Link",
+                    help="Click to visit the website",
+                    width="small",
+                    display_text="Visit Site"
+                ),
+            },
+            hide_index=True,
+            use_container_width=True,
+        )
 
-    # Display the table in Streamlit
-    st.table(table_data)
-
-    # Display the 'answer' in Streamlit
-    answer = output_data.get("answer", "No answer available")
-    st.write(f"**The answer to your search query:** {answer}")
-
-    # Display follow-up questions if available
-    follow_up_questions = output_data.get("follow_up_questions", [])
-    if follow_up_questions:
-        st.write(f"**Follow-up questions for the query:** {output_data.get('query')}")
-        st.write(", ".join(follow_up_questions))
+        # Add popovers for full content display
+        for item in output_data.get("results", []):
+            with st.popover(f"View content: {item.get('title', '')[:50]}..."):
+                st.markdown(item.get("content", ""))
+    else:
+        st.info("No results found for your search query.")
 
 
 def print_result_table(output_data):

lib/alwrity_ui/display_google_serp_results.py

@@ -0,0 +1,277 @@
+import streamlit as st
+import logging
+from datetime import datetime
+from typing import Dict, Optional, Any
+
+# Configure module logger
+logger = logging.getLogger(__name__)
+
+def display_research_results(results: Dict[str, Any]) -> None:
+    """
+    Display research results in a structured format with tabs.
+    
+    Args:
+        results (dict): Processed research results containing summary and data
+    """
+    if not results:
+        st.warning("No results to display")
+        return
+
+    # Create tabs for different result sections
+    tabs = st.tabs(["📊 Summary", "🔍 Results", "📈 Statistics"])
+    
+    with tabs[0]:
+        display_summary_section(results)
+    
+    with tabs[1]:
+        if results['source'] == 'gemini':
+            display_gemini_results(results)
+        else:
+            display_serp_results(results)
+    
+    with tabs[2]:
+        display_statistics(results)
+
+def process_research_results(results: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """Process and format research results."""
+    logger.info("Processing research results")
+    
+    try:
+        if not results:
+            return None
+            
+        processed = {
+            'timestamp': str(datetime.now()),
+            'source': results.get('source', 'unknown'),
+            'summary': {},
+            'data': {}
+        }
+        
+        if results.get('source') == 'gemini':
+            processed.update(process_gemini_results(results))
+        else:
+            processed.update(process_serp_results(results))
+        
+        logger.info("Results processing completed")
+        return processed
+        
+    except Exception as err:
+        logger.error(f"Failed to process results: {err}", exc_info=True)
+        return None
+
+def process_search_results(search_results: Dict[str, Any], search_type: str = "general") -> Optional[Dict[str, Any]]:
+    """Process search results and prepare for display."""
+    logger.info(f"Processing {search_type} search results")
+    
+    try:
+        if not search_results:
+            return None
+            
+        processed = {
+            'organic': process_organic_results(search_results.get('organic', [])),
+            'peopleAlsoAsk': process_paa_results(search_results.get('peopleAlsoAsk', [])),
+            'relatedSearches': process_related_searches(search_results.get('relatedSearches', [])),
+            'metadata': {
+                'timestamp': str(datetime.now()),
+                'type': search_type
+            }
+        }
+        
+        return processed
+        
+    except Exception as err:
+        logger.error(f"Error processing search results: {err}", exc_info=True)
+        return None
+
+# Helper functions for result processing
+def process_organic_results(results):
+    """Process organic search results."""
+    return [{
+        'title': result.get('title', 'No Title'),
+        'link': result.get('link', '#'),
+        'snippet': result.get('snippet', 'No snippet available'),
+        'position': result.get('position', 'N/A')
+    } for result in results]
+
+def process_paa_results(results):
+    """Process People Also Ask results."""
+    return [{
+        'question': result.get('title', ''),
+        'answer': result.get('snippet', 'No answer available'),
+        'link': result.get('link', '#')
+    } for result in results]
+
+def process_related_searches(results):
+    """Process related searches."""
+    return [query.get('query', '') for query in results]
+
+def process_gemini_results(results: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Process Gemini API research results.
+    
+    Args:
+        results (dict): Raw Gemini research results
+        
+    Returns:
+        dict: Processed results with summary and data
+    """
+    gemini_data = results.get('results', {})
+    return {
+        'summary': {
+            'main_findings': gemini_data.get('main_response', ''),
+            'sources': gemini_data.get('grounding_data', []),
+            'processing_time': gemini_data.get('metadata', {}).get('timestamp'),
+            'total_sources': len(gemini_data.get('grounding_data', [])),
+            'model': gemini_data.get('metadata', {}).get('model', 'unknown')
+        },
+        'data': gemini_data
+    }
+
+def process_serp_results(results: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Process SERP search results.
+    
+    Args:
+        results (dict): Raw SERP results
+        
+    Returns:
+        dict: Processed results with summary and data
+    """
+    organic_results = results.get('organic', [])
+    paa_results = results.get('peopleAlsoAsk', [])
+    related_searches = results.get('relatedSearches', [])
+    
+    return {
+        'summary': {
+            'total_results': len(organic_results),
+            'sources': [result.get('link') for result in organic_results],
+            'titles': [result.get('title') for result in organic_results],
+            'total_questions': len(paa_results),
+            'total_related': len(related_searches)
+        },
+        'data': {
+            'organic': process_organic_results(organic_results),
+            'peopleAlsoAsk': process_paa_results(paa_results),
+            'relatedSearches': process_related_searches(related_searches)
+        }
+    }
+
+# Display helper functions
+def display_summary_section(results):
+    """Display summary section of results."""
+    st.markdown("### 📋 Research Summary")
+    st.markdown(f"""
+    - **Source**: {results['source'].title()}
+    - **Time**: {results['timestamp']}
+    - **Total Sources**: {len(results.get('summary', {}).get('sources', []))}
+    """)
+
+def display_gemini_results(results):
+    """Display Gemini-specific results."""
+    st.markdown("### 🤖 Gemini Research Findings")
+    st.write(results['summary']['main_findings'])
+    
+    with st.expander("🌐 Sources and References", expanded=False):
+        st.write(results['data'].get('grounding_data', 'No sources available'))
+
+def display_serp_results(results):
+    """Display SERP-specific results."""
+    st.markdown("### 🔍 Search Results")
+    
+    for result in results['data'].get('organic', []):
+        with st.expander(f"📄 {result['title']}", expanded=False):
+            st.markdown(f"""
+            **Rank:** {result['position']}
+            
+            **Link:** [{result['link']}]({result['link']})
+            
+            **Snippet:**
+            {result['snippet']}
+            """)
+
+def display_statistics(results: Dict[str, Any]) -> None:
+    """
+    Display statistical information about search results.
+    
+    Args:
+        results (dict): Processed research results
+    """
+    st.markdown("### 📈 Research Statistics")
+    
+    # Source-specific metrics
+    if results['source'] == 'gemini':
+        col1, col2 = st.columns(2)
+        with col1:
+            st.metric(
+                "Sources Analyzed",
+                results.get('summary', {}).get('total_sources', 0)
+            )
+        with col2:
+            st.metric(
+                "Model Used",
+                results.get('summary', {}).get('model', 'Unknown')
+            )
+            
+    else:  # SERP results
+        col1, col2, col3 = st.columns(3)
+        with col1:
+            st.metric(
+                "Organic Results",
+                results.get('summary', {}).get('total_results', 0)
+            )
+        with col2:
+            st.metric(
+                "Related Questions",
+                results.get('summary', {}).get('total_questions', 0)
+            )
+        with col3:
+            st.metric(
+                "Related Searches",
+                results.get('summary', {}).get('total_related', 0)
+            )
+    
+    # Common metrics
+    st.markdown("#### 🕒 Timing Information")
+    st.info(f"Research completed at: {results['timestamp']}")
+    
+    # Display data quality metrics
+    st.markdown("#### 📊 Data Quality")
+    quality_metrics = calculate_quality_metrics(results)
+    
+    col1, col2 = st.columns(2)
+    with col1:
+        st.progress(quality_metrics['completeness'])
+        st.caption("Data Completeness")
+    with col2:
+        st.progress(quality_metrics['relevance'])
+        st.caption("Estimated Relevance")
+
+def calculate_quality_metrics(results: Dict[str, Any]) -> Dict[str, float]:
+    """
+    Calculate quality metrics for the research results.
+    
+    Args:
+        results (dict): Processed research results
+        
+    Returns:
+        dict: Quality metrics including completeness and relevance scores
+    """
+    try:
+        if results['source'] == 'gemini':
+            completeness = 1.0 if results['summary']['main_findings'] else 0.0
+            relevance = 0.8 if results['summary']['sources'] else 0.4
+        else:
+            organic_results = results.get('summary', {}).get('total_results', 0)
+            completeness = min(organic_results / 10, 1.0)  # Normalize to 0-1
+            has_paa = bool(results.get('summary', {}).get('total_questions', 0))
+            has_related = bool(results.get('summary', {}).get('total_related', 0))
+            relevance = (0.6 + (0.2 if has_paa else 0) + (0.2 if has_related else 0))
+        
+        return {
+            'completeness': completeness,
+            'relevance': relevance
+        }
+        
+    except Exception as err:
+        logger.error(f"Error calculating quality metrics: {err}")
+        return {'completeness': 0.0, 'relevance': 0.0}

lib/alwrity_ui/google_trends_ui.py

@@ -0,0 +1,458 @@
+"""
+Module for displaying Google Trends data in the Streamlit UI.
+
+This module provides functions for visualizing Google Trends data, including:
+- Interest over time
+- Regional interest
+- Related queries
+- Related topics
+"""
+
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+import logging
+
+# Set up logging
+logger = logging.getLogger(__name__)
+
+def display_google_trends_data(trends_data, search_keyword):
+    """
+    Display Google Trends data in a structured format with tabs for different sections.
+    
+    Args:
+        trends_data (dict): Dictionary containing Google Trends data
+        search_keyword (str): The search keyword used for the analysis
+    """
+    if not trends_data:
+        st.warning("No Google Trends data available for this search.")
+        return
+        
+    st.subheader(f"Google Trends Analysis for '{search_keyword}'")
+    
+    # Add an informative message about Google Trends
+    with st.expander("ℹ️ About Google Trends Data", expanded=False):
+        st.markdown("""
+        **What is Google Trends?**
+        
+        Google Trends is a public web facility that shows how often a particular search-term is entered relative to the total search-volume across various regions of the world, and in various languages.
+        
+        **What data is shown here?**
+        
+        - **Related Keywords**: Terms that are frequently searched together with your keyword
+        - **Interest Over Time**: How interest in your keyword has changed over the past 12 months
+        - **Regional Interest**: Where in the world your keyword is most popular
+        - **Related Queries**: What people search for before and after searching for your keyword
+        - **Related Topics**: Topics that are closely related to your keyword
+        
+        **How to interpret the data:**
+        
+        - Interest values range from 0 to 100, where 100 is the peak popularity for the term
+        - A value of 50 means the term is half as popular as the peak
+        - A value of 0 means there was not enough data for this term
+        """)
+    
+    # Create tabs for different sections
+    tab1, tab2, tab3, tab4, tab5 = st.tabs([
+        "Related Keywords", 
+        "Interest Over Time", 
+        "Regional Interest", 
+        "Related Queries", 
+        "Related Topics"
+    ])
+    
+    with tab1:
+        display_keywords_section(trends_data.get('related_keywords', []))
+        
+    with tab2:
+        display_interest_over_time(trends_data.get('interest_over_time', pd.DataFrame()))
+        
+    with tab3:
+        display_regional_interest(trends_data.get('regional_interest', pd.DataFrame()))
+        
+    with tab4:
+        display_related_queries(trends_data.get('related_queries', pd.DataFrame()))
+        
+    with tab5:
+        display_related_topics(trends_data.get('related_topics', pd.DataFrame()))
+        
+    # Add a footer with data source information
+    st.markdown("---")
+    st.caption("Data source: Google Trends | Last updated: " + pd.Timestamp.now().strftime("%Y-%m-%d %H:%M:%S"))
+
+def display_keywords_section(keywords):
+    """Display related keywords from Google Trends in a table format."""
+    if not keywords:
+        st.info("No related keywords data available.")
+        return
+        
+    st.subheader("Related Keywords")
+    st.write("Keywords related to your search:")
+    
+    # Add explanation about related keywords
+    with st.expander("ℹ️ About Related Keywords", expanded=False):
+        st.markdown("""
+        **What are Related Keywords?**
+        
+        Related keywords are terms that are frequently searched together with your main keyword. 
+        These keywords can help you understand what topics are associated with your search term 
+        and can be valuable for content planning and SEO strategies.
+        
+        **How to use this data:**
+        
+        - Use these keywords to expand your content strategy
+        - Identify gaps in your content that you could fill
+        - Understand what your audience is interested in
+        - Improve your SEO by incorporating these terms naturally in your content
+        """)
+    
+    # Create a DataFrame for better display
+    df = pd.DataFrame(keywords, columns=['Keyword'])
+    st.dataframe(df, use_container_width=True)
+    
+    # Add a note about the number of keywords
+    st.caption(f"Found {len(keywords)} related keywords")
+
+def display_interest_over_time(interest_df):
+    """Display a chart showing interest over time for a given search keyword."""
+    if interest_df.empty:
+        st.info("No interest over time data available.")
+        return
+        
+    st.subheader("Interest Over Time")
+    
+    # Add explanation about interest over time
+    with st.expander("ℹ️ About Interest Over Time", expanded=False):
+        st.markdown("""
+        **What is Interest Over Time?**
+        
+        Interest Over Time shows how interest in your search term has changed over the past 12 months.
+        The data is normalized and presented on a scale from 0 to 100, where 100 is the peak popularity 
+        for the term, 50 means the term is half as popular, and 0 means there was not enough data.
+        
+        **How to interpret this chart:**
+        
+        - Look for peaks and valleys to identify trends
+        - Compare with seasonal patterns or events
+        - Identify if interest is growing, declining, or stable
+        - Use this data to time your content releases for maximum impact
+        """)
+    
+    try:
+        # Ensure we have the required columns
+        if 'date' not in interest_df.columns:
+            st.error("Interest over time data is missing the 'date' column.")
+            return
+            
+        if 'interest' not in interest_df.columns:
+            st.error("Interest over time data is missing the 'interest' column.")
+            return
+            
+        # Create the chart
+        fig = px.line(
+            interest_df, 
+            x='date', 
+            y='interest',
+            title='Interest Over Time',
+            labels={'date': 'Date', 'interest': 'Interest'},
+            line_shape='spline'
+        )
+        
+        fig.update_layout(
+            xaxis_title="Date",
+            yaxis_title="Interest",
+            hovermode='x unified'
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)
+        
+        # Add summary statistics
+        if not interest_df.empty:
+            col1, col2, col3 = st.columns(3)
+            with col1:
+                st.metric("Average Interest", f"{interest_df['interest'].mean():.1f}")
+            with col2:
+                st.metric("Peak Interest", f"{interest_df['interest'].max():.1f}")
+            with col3:
+                st.metric("Lowest Interest", f"{interest_df['interest'].min():.1f}")
+        
+    except Exception as e:
+        st.error(f"Error displaying interest over time chart: {str(e)}")
+        logger.error(f"Error in display_interest_over_time: {e}")
+
+def display_regional_interest(regional_df):
+    """Display a chart showing interest by region for the search keyword."""
+    if regional_df.empty:
+        st.info("No regional interest data available.")
+        return
+        
+    st.subheader("Regional Interest")
+    
+    # Add explanation about regional interest
+    with st.expander("ℹ️ About Regional Interest", expanded=False):
+        st.markdown("""
+        **What is Regional Interest?**
+        
+        Regional Interest shows how interest in your search term varies across different countries.
+        The data is normalized and presented on a scale from 0 to 100, where 100 is the peak popularity 
+        for the term in that region, 50 means the term is half as popular, and 0 means there was not enough data.
+        
+        **How to interpret this map:**
+        
+        - Darker colors indicate higher interest in that region
+        - Lighter colors indicate lower interest
+        - Hover over a country to see the exact interest value
+        - Use this data to target your content to specific regions
+        """)
+    
+    try:
+        # Ensure we have the required columns
+        if 'country_code' not in regional_df.columns:
+            st.error("Regional interest data is missing the 'country_code' column.")
+            return
+            
+        if 'interest' not in regional_df.columns:
+            st.error("Regional interest data is missing the 'interest' column.")
+            return
+            
+        # Create the choropleth map
+        fig = go.Figure(data=go.Choropleth(
+            locations=regional_df['country_code'],
+            z=regional_df['interest'],
+            text=regional_df['country_code'],  # This will show in the hover text
+            colorscale='Viridis',
+            colorbar_title="Interest Level",
+            zmin=0,
+            zmax=100,
+            marker_line_color='darkgray',
+            marker_line_width=0.5,
+            showscale=True,
+            colorbar=dict(
+                title="Interest Level",
+                tickformat=".0f",
+                tickmode="linear",
+                tick0=0,
+                dtick=20
+            )
+        ))
+        
+        # Update the layout for better visualization
+        fig.update_layout(
+            title=dict(
+                text='Regional Interest Distribution',
+                x=0.5,
+                xanchor='center'
+            ),
+            geo=dict(
+                showframe=False,
+                showcoastlines=True,
+                projection_type='equirectangular',
+                showland=True,
+                landcolor='lightgray',
+                showocean=True,
+                oceancolor='aliceblue',
+                showcountries=True,
+                countrycolor='darkgray'
+            ),
+            width=800,
+            height=500,
+            margin=dict(l=0, r=0, t=30, b=0)
+        )
+        
+        # Display the map
+        st.plotly_chart(fig, use_container_width=True)
+        
+        # Display top 5 countries with highest interest
+        if not regional_df.empty:
+            st.subheader("Top Regions by Interest")
+            top_regions = regional_df.sort_values('interest', ascending=False).head(5)
+            
+            # Create a more visually appealing bar chart for top regions
+            fig_bar = go.Figure(data=[
+                go.Bar(
+                    x=top_regions['country_code'],
+                    y=top_regions['interest'],
+                    text=top_regions['interest'].round(1),
+                    textposition='auto',
+                    marker_color='rgb(55, 83, 109)'
+                )
+            ])
+            
+            fig_bar.update_layout(
+                title='Top 5 Regions by Interest Level',
+                xaxis_title='Region',
+                yaxis_title='Interest Level',
+                yaxis_range=[0, 100],
+                showlegend=False
+            )
+            
+            st.plotly_chart(fig_bar, use_container_width=True)
+        
+    except Exception as e:
+        st.error(f"Error displaying regional interest chart: {str(e)}")
+        logger.error(f"Error in display_regional_interest: {e}")
+
+def display_related_queries(queries_df):
+    """Display related queries in a structured format."""
+    if queries_df.empty:
+        st.info("No related queries data available.")
+        return
+        
+    st.subheader("Related Queries")
+    
+    # Add explanation about related queries
+    with st.expander("ℹ️ About Related Queries", expanded=False):
+        st.markdown("""
+        **What are Related Queries?**
+        
+        Related Queries show what people search for before and after searching for your keyword.
+        These queries can help you understand the search intent and context around your keyword.
+        
+        **How to interpret this data:**
+        
+        - The 'value' column shows the relative interest compared to your main keyword
+        - Higher values indicate stronger association with your keyword
+        - Use these queries to expand your content strategy
+        - Identify what questions your audience is trying to answer
+        """)
+    
+    try:
+        # Ensure we have the required columns
+        if 'query' not in queries_df.columns:
+            st.error("Related queries data is missing the 'query' column.")
+            return
+            
+        if 'value' not in queries_df.columns:
+            st.error("Related queries data is missing the 'value' column.")
+            return
+            
+        # Sort by value in descending order
+        queries_df = queries_df.sort_values('value', ascending=False)
+        
+        # Display as a table
+        st.dataframe(queries_df, use_container_width=True)
+        
+        # Add a note about the number of queries
+        st.caption(f"Found {len(queries_df)} related queries")
+        
+    except Exception as e:
+        st.error(f"Error displaying related queries: {str(e)}")
+        logger.error(f"Error in display_related_queries: {e}")
+
+def display_trending_searches(trending_df):
+    """Display trending searches in the UI."""
+    if trending_df.empty:
+        st.info("No trending searches data available.")
+        return
+        
+    st.subheader("📊 Trending Searches")
+    
+    # Display as numbered list with emojis
+    for idx, search in enumerate(trending_df[0].head(10), 1):
+        st.write(f"{idx}. 🔍 {search}")
+
+def display_realtime_trends(trends_df):
+    """Display realtime trending searches in the UI."""
+    if trends_df.empty:
+        st.info("No realtime trends data available.")
+        return
+        
+    st.subheader("⚡ Realtime Trends")
+    
+    # Create tabs for different categories
+    if not trends_df.empty:
+        # Display top 5 trends with their titles and articles
+        for _, row in trends_df.head(5).iterrows():
+            with st.expander(f"🔥 {row.get('title', 'Trending Topic')}"):
+                st.write(f"**Traffic:** {row.get('traffic', 'N/A')}")
+                if 'articles' in row:
+                    st.write("📰 Related Articles:")
+                    for article in row['articles'][:3]:  # Show top 3 articles
+                        st.write(f"- {article['title']}")
+
+
+
+def display_related_topics(topics_df):
+    """Display related topics in a structured format."""
+    if topics_df.empty:
+        st.info("No related topics data available.")
+        return
+        
+    st.subheader("Related Topics")
+    
+    # Add explanation about related topics
+    with st.expander("ℹ️ About Related Topics", expanded=False):
+        st.markdown("""
+        **What are Related Topics?**
+        
+        Related Topics show broader topics that are associated with your search term.
+        These topics can help you understand the broader context and themes related to your keyword.
+        
+        **How to interpret this data:**
+        
+        - The 'value' column shows the relative interest compared to your main keyword
+        - Higher values indicate stronger association with your keyword
+        - Use these topics to understand the broader context of your keyword
+        - Identify themes that might be relevant to your content strategy
+        """)
+    
+    try:
+        # Ensure we have the required columns
+        if 'topic' not in topics_df.columns:
+            st.error("Related topics data is missing the 'topic' column.")
+            return
+            
+        if 'value' not in topics_df.columns:
+            st.error("Related topics data is missing the 'value' column.")
+            return
+            
+        # Sort by value in descending order
+        topics_df = topics_df.sort_values('value', ascending=False)
+        
+        # Display as a table
+        st.dataframe(topics_df, use_container_width=True)
+        
+        # Add a note about the number of topics
+        st.caption(f"Found {len(topics_df)} related topics")
+        
+    except Exception as e:
+        st.error(f"Error displaying related topics: {str(e)}")
+        logger.error(f"Error in display_related_topics: {e}")
+
+def process_trends_data(trends_data):
+    """
+    Process and format Google Trends data for display.
+    
+    Args:
+        trends_data (dict): Raw Google Trends data
+        
+    Returns:
+        dict: Formatted data ready for display
+    """
+    if not trends_data:
+        return {}
+        
+    processed_data = {}
+    
+    # Process related keywords
+    if 'related_keywords' in trends_data:
+        processed_data['related_keywords'] = trends_data['related_keywords']
+        
+    # Process interest over time
+    if 'interest_over_time' in trends_data and not trends_data['interest_over_time'].empty:
+        processed_data['interest_over_time'] = trends_data['interest_over_time']
+        
+    # Process regional interest
+    if 'regional_interest' in trends_data and not trends_data['regional_interest'].empty:
+        processed_data['regional_interest'] = trends_data['regional_interest']
+        
+    # Process related queries
+    if 'related_queries' in trends_data and not trends_data['related_queries'].empty:
+        processed_data['related_queries'] = trends_data['related_queries']
+        
+    # Process related topics
+    if 'related_topics' in trends_data and not trends_data['related_topics'].empty:
+        processed_data['related_topics'] = trends_data['related_topics']
+        
+    return processed_data 

lib/alwrity_ui/keyword_web_researcher.py

@@ -0,0 +1,529 @@
+import os
+import time
+import logging
+import streamlit as st
+from datetime import datetime
+
+from lib.ai_web_researcher.gpt_online_researcher import gpt_web_researcher
+from lib.utils.read_main_config_params import read_return_config_section
+
+# Configure module-level logging
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+# Create console handler if it doesn't exist
+if not logger.handlers:
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.DEBUG)
+    formatter = logging.Formatter(
+        '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S'
+    )
+    console_handler.setFormatter(formatter)
+    logger.addHandler(console_handler)
+
+def reload_env_variables():
+    """Reload environment variables from .env file."""
+    try:
+        from dotenv import load_dotenv
+        load_dotenv(override=True)
+        return True
+    except Exception as e:
+        logger.error(f"Failed to reload environment variables: {str(e)}")
+        return False
+
+def save_api_key_to_env(key_name, key_value):
+    """Save API key to .env file."""
+    try:
+        env_path = os.path.join(os.getcwd(), '.env')
+        
+        # Read existing .env content
+        existing_content = {}
+        if os.path.exists(env_path):
+            with open(env_path, 'r') as f:
+                for line in f:
+                    if '=' in line:
+                        key, value = line.strip().split('=', 1)
+                        existing_content[key] = value
+        
+        # Update or add new key
+        existing_content[key_name] = key_value
+        
+        # Write back to .env
+        with open(env_path, 'w') as f:
+            for key, value in existing_content.items():
+                f.write(f"{key}={value}\n")
+        
+        # Update environment variable and reload all env vars
+        os.environ[key_name] = key_value
+        if reload_env_variables():
+            return True
+        return False
+    except Exception as e:
+        logger.error(f"Failed to save API key to .env: {str(e)}")
+        return False
+
+def validate_api_keys():
+    """Validate required API keys and return their status."""
+    
+    logger.info("Validating API keys")
+    
+    # Get API keys
+    api_keys = {
+        'SERPER_API_KEY': os.getenv('SERPER_API_KEY'),
+        'METAPHOR_API_KEY': os.getenv('METAPHOR_API_KEY'),
+        'TAVILY_API_KEY': os.getenv('TAVILY_API_KEY'),
+        'FIRECRAWL_API_KEY': os.getenv('FIRECRAWL_API_KEY')
+    }
+    
+    # Test SERPER_API_KEY validity
+    if api_keys['SERPER_API_KEY']:
+        try:
+            # Make a test request
+            import requests
+            test_url = "https://google.serper.dev/search"
+            headers = {
+                'X-API-KEY': api_keys['SERPER_API_KEY'],
+                'Content-Type': 'application/json'
+            }
+            test_payload = {"q": "test", "gl": "us", "hl": "en", "num": 1}
+            
+            response = requests.post(test_url, headers=headers, json=test_payload)
+            api_keys['SERPER_API_KEY_VALID'] = response.status_code == 200
+            
+            if not api_keys['SERPER_API_KEY_VALID']:
+                logger.error(f"SERPER_API_KEY validation failed: {response.status_code} - {response.text}")
+        except Exception as e:
+            logger.error(f"Error validating SERPER_API_KEY: {str(e)}")
+            api_keys['SERPER_API_KEY_VALID'] = False
+    else:
+        api_keys['SERPER_API_KEY_VALID'] = False
+
+    return api_keys
+
+def do_web_research():
+    """Main function to perform web research based on user input."""
+    
+    # Reset session state variables for this research operation
+    if 'metaphor_results_displayed' in st.session_state:
+        del st.session_state.metaphor_results_displayed
+    
+    logger.info("Starting do_web_research function")
+    
+    try:
+        # Get API keys without validation
+        api_keys = {
+            'SERPER_API_KEY': os.getenv('SERPER_API_KEY'),
+            'METAPHOR_API_KEY': os.getenv('METAPHOR_API_KEY'),
+            'TAVILY_API_KEY': os.getenv('TAVILY_API_KEY'),
+            'FIRECRAWL_API_KEY': os.getenv('FIRECRAWL_API_KEY')
+        }
+        
+        if not api_keys['SERPER_API_KEY']:
+            st.error("""
+            🚫 SERPER_API_KEY is missing. Please configure your API key.
+            """)
+            with st.popover("⚙️ Configure API Keys"):
+                st.markdown("""
+                ### API Key Configuration
+                Enter your API keys below to enable research features.
+                """)
+                
+                # SERPER API Key
+                serper_col1, serper_col2 = st.columns([3, 1])
+                with serper_col1:
+                    serper_key = st.text_input(
+                        "Serper API Key",
+                        type="password",
+                        placeholder="Enter your Serper API key",
+                        help="Get your key at https://serper.dev"
+                    )
+                    test_key = st.checkbox("Test API key before saving", value=False, help="Validate the API key before saving")
+                with serper_col2:
+                    if st.button("Save Serper", use_container_width=True):
+                        if serper_key:
+                            if test_key:
+                                # Test the API key
+                                try:
+                                    import requests
+                                    test_url = "https://google.serper.dev/search"
+                                    headers = {
+                                        'X-API-KEY': serper_key,
+                                        'Content-Type': 'application/json'
+                                    }
+                                    test_payload = {"q": "test", "gl": "us", "hl": "en", "num": 1}
+                                    response = requests.post(test_url, headers=headers, json=test_payload)
+                                    
+                                    if response.status_code == 200:
+                                        if save_api_key_to_env('SERPER_API_KEY', serper_key):
+                                            st.success("✅ Serper API key validated and saved!")
+                                            st.rerun()
+                                        else:
+                                            st.error("Failed to save API key")
+                                    else:
+                                        st.error(f"API key validation failed: {response.status_code} - {response.text}")
+                                except Exception as e:
+                                    st.error(f"Error validating API key: {str(e)}")
+                            else:
+                                # Skip validation and save directly
+                                if save_api_key_to_env('SERPER_API_KEY', serper_key):
+                                    st.success("✅ Serper API key saved!")
+                                    time.sleep(0.5)  # Small delay to ensure the key is saved
+                                    st.rerun()
+                                else:
+                                    st.error("Failed to save API key")
+                
+                # METAPHOR API Key
+                if not api_keys.get('METAPHOR_API_KEY'):
+                    metaphor_col1, metaphor_col2 = st.columns([3, 1])
+                    with metaphor_col1:
+                        metaphor_key = st.text_input(
+                            "Metaphor API Key",
+                            type="password",
+                            placeholder="Enter your Metaphor API key",
+                            help="Get your key at https://metaphor.systems"
+                        )
+                        test_metaphor = st.checkbox("Test API key before saving", value=False, help="Validate the API key before saving")
+                    with metaphor_col2:
+                        if st.button("Save Metaphor", use_container_width=True):
+                            if metaphor_key:
+                                if test_metaphor:
+                                    # Test the API key
+                                    try:
+                                        import requests
+                                        test_url = "https://api.metaphor.systems/v1/search"
+                                        headers = {
+                                            'Authorization': f'Bearer {metaphor_key}',
+                                            'Content-Type': 'application/json'
+                                        }
+                                        test_payload = {"query": "test", "numResults": 1}
+                                        response = requests.post(test_url, headers=headers, json=test_payload)
+                                        
+                                        if response.status_code == 200:
+                                            if save_api_key_to_env('METAPHOR_API_KEY', metaphor_key):
+                                                st.success("✅ Metaphor API key validated and saved!")
+                                                st.rerun()
+                                            else:
+                                                st.error("Failed to save API key")
+                                        else:
+                                            st.error(f"API key validation failed: {response.status_code} - {response.text}")
+                                    except Exception as e:
+                                        st.error(f"Error validating API key: {str(e)}")
+                                else:
+                                    # Skip validation and save directly
+                                    if save_api_key_to_env('METAPHOR_API_KEY', metaphor_key):
+                                        st.success("✅ Metaphor API key saved!")
+                                        st.rerun()
+                                    else:
+                                        st.error("Failed to save API key")
+                
+                # TAVILY API Key
+                if not api_keys.get('TAVILY_API_KEY'):
+                    tavily_col1, tavily_col2 = st.columns([3, 1])
+                    with tavily_col1:
+                        tavily_key = st.text_input(
+                            "Tavily API Key",
+                            type="password",
+                            placeholder="Enter your Tavily API key",
+                            help="Get your key at https://tavily.com"
+                        )
+                        test_tavily = st.checkbox("Test API key before saving", value=False, help="Validate the API key before saving")
+                    with tavily_col2:
+                        if st.button("Save Tavily", use_container_width=True):
+                            if tavily_key:
+                                if test_tavily:
+                                    # Test the API key
+                                    try:
+                                        import requests
+                                        test_url = "https://api.tavily.com/v1/search"
+                                        headers = {
+                                            'Authorization': f'Bearer {tavily_key}',
+                                            'Content-Type': 'application/json'
+                                        }
+                                        test_payload = {"query": "test", "max_results": 1}
+                                        response = requests.post(test_url, headers=headers, json=test_payload)
+                                        
+                                        if response.status_code == 200:
+                                            if save_api_key_to_env('TAVILY_API_KEY', tavily_key):
+                                                st.success("✅ Tavily API key validated and saved!")
+                                                st.rerun()
+                                            else:
+                                                st.error("Failed to save API key")
+                                        else:
+                                            st.error(f"API key validation failed: {response.status_code} - {response.text}")
+                                    except Exception as e:
+                                        st.error(f"Error validating API key: {str(e)}")
+                                else:
+                                    # Skip validation and save directly
+                                    if save_api_key_to_env('TAVILY_API_KEY', tavily_key):
+                                        st.success("✅ Tavily API key saved!")
+                                        st.rerun()
+                                    else:
+                                        st.error("Failed to save API key")
+                
+                # FIRECRAWL API Key
+                if not api_keys.get('FIRECRAWL_API_KEY'):
+                    firecrawl_col1, firecrawl_col2 = st.columns([3, 1])
+                    with firecrawl_col1:
+                        firecrawl_key = st.text_input(
+                            "Firecrawl API Key",
+                            type="password",
+                            placeholder="Enter your Firecrawl API key",
+                            help="Get your key at https://firecrawl.co"
+                        )
+                        test_firecrawl = st.checkbox("Test API key before saving", value=False, help="Validate the API key before saving")
+                    with firecrawl_col2:
+                        if st.button("Save Firecrawl", use_container_width=True):
+                            if firecrawl_key:
+                                if test_firecrawl:
+                                    # Test the API key
+                                    try:
+                                        import requests
+                                        test_url = "https://api.firecrawl.co/v1/search"
+                                        headers = {
+                                            'Authorization': f'Bearer {firecrawl_key}',
+                                            'Content-Type': 'application/json'
+                                        }
+                                        test_payload = {"query": "test", "limit": 1}
+                                        response = requests.post(test_url, headers=headers, json=test_payload)
+                                        
+                                        if response.status_code == 200:
+                                            if save_api_key_to_env('FIRECRAWL_API_KEY', firecrawl_key):
+                                                st.success("✅ Firecrawl API key validated and saved!")
+                                                st.rerun()
+                                            else:
+                                                st.error("Failed to save API key")
+                                        else:
+                                            st.error(f"API key validation failed: {response.status_code} - {response.text}")
+                                    except Exception as e:
+                                        st.error(f"Error validating API key: {str(e)}")
+                                else:
+                                    # Skip validation and save directly
+                                    if save_api_key_to_env('FIRECRAWL_API_KEY', firecrawl_key):
+                                        st.success("✅ Firecrawl API key saved!")
+                                        st.rerun()
+                                    else:
+                                        st.error("Failed to save API key")
+                
+                st.markdown("""
+                ---
+                ### Need Help?
+                1. Click the links above to get your API keys
+                2. Enter the keys in the fields above
+                3. Click Save to store them securely
+                4. The app will refresh automatically
+                """)
+            return
+
+        # Initialize session state for research options
+        if "research_options" not in st.session_state:
+            st.session_state.research_options = {
+                "primary_keywords": "",
+                "related_keywords": "",
+                "target_audience": ["General"],
+                "content_type": ["Blog Posts"],
+                "search_depth": 3,
+                "geo_location": "us",
+                "search_language": "en",
+                "num_results": 10,
+                "time_range": "past month",
+                "include_domains": "",
+                "similar_url": "",
+                "search_mode": "google"  # Default search mode
+            }
+
+        # Define the research options dialog function
+        @st.dialog("🔍 Research Options", width="large")
+        def show_research_options():
+            tab1, tab2, tab3 = st.tabs(["Basic", "Advanced", "Technical"])
+            
+            with tab1:
+                st.session_state.research_options["related_keywords"] = st.text_input(
+                    "Related Keywords",
+                    value=st.session_state.research_options["related_keywords"],
+                    placeholder="Enter related terms...",
+                    help="Additional keywords to provide context and expand research"
+                )
+                
+                st.session_state.research_options["target_audience"] = st.multiselect(
+                    "Target Audience",
+                    ["General", "Technical", "Business", "Academic", "Youth", "Senior"],
+                    default=st.session_state.research_options["target_audience"],
+                    help="Select your target audience to focus research"
+                )
+                
+                st.session_state.research_options["content_type"] = st.multiselect(
+                    "Content Type",
+                    ["Blog Posts", "Articles", "Social Media", "Whitepapers", "Tutorials", "Videos"],
+                    default=st.session_state.research_options["content_type"],
+                    help="Select content types to tailor research results"
+                )
+                
+                st.session_state.research_options["search_depth"] = st.slider(
+                    "Search Depth",
+                    min_value=1,
+                    max_value=5,
+                    value=st.session_state.research_options["search_depth"],
+                    help="Higher depth means more comprehensive but slower research"
+                )
+
+            with tab2:
+                col1, col2 = st.columns(2)
+                with col1:
+                    st.session_state.research_options["geo_location"] = st.selectbox(
+                        "Geographic Location",
+                        options=["us", "in", "uk", "fr", "de", "jp", "custom"],
+                        index=["us", "in", "uk", "fr", "de", "jp"].index(st.session_state.research_options["geo_location"]),
+                        help="Target specific geographic region for research"
+                    )
+                    
+                    st.session_state.research_options["num_results"] = st.number_input(
+                        "Number of Results",
+                        min_value=1,
+                        max_value=100,
+                        value=st.session_state.research_options["num_results"],
+                        help="Number of results to analyze"
+                    )
+                
+                with col2:
+                    st.session_state.research_options["search_language"] = st.selectbox(
+                        "Search Language",
+                        options=["en", "hi", "fr", "de", "es", "custom"],
+                        index=["en", "hi", "fr", "de", "es"].index(st.session_state.research_options["search_language"]),
+                        help="Primary language for search results"
+                    )
+                    
+                    st.session_state.research_options["time_range"] = st.selectbox(
+                        "Time Range",
+                        options=["past day", "past week", "past month", "past year", "anytime"],
+                        index=["past day", "past week", "past month", "past year", "anytime"].index(st.session_state.research_options["time_range"]),
+                        help="Time period for research results"
+                    )
+
+            with tab3:
+                st.session_state.research_options["include_domains"] = st.text_input(
+                    "Include Domains",
+                    value=st.session_state.research_options["include_domains"],
+                    placeholder="example.com, another.com",
+                    help="Specific domains to include in research"
+                )
+                
+                st.session_state.research_options["similar_url"] = st.text_input(
+                    "Similar URL",
+                    value=st.session_state.research_options["similar_url"],
+                    placeholder="https://example.com/page",
+                    help="Find content similar to this URL"
+                )
+
+                # Research method selection
+                st.markdown("### Select Research Method")
+                search_options = [
+                    ("google", "🔍 Google Search", "Traditional web research with AI analysis", bool(api_keys['SERPER_API_KEY'])),
+                    ("ai", "🤖 AI Search", "Neural search with semantic analysis", bool(api_keys['METAPHOR_API_KEY'] and api_keys['TAVILY_API_KEY'])),
+                    ("deep", "🔬 Deep Search (Beta)", "Advanced deep web analysis", bool(all(api_keys.values())))
+                ]
+                
+                enabled_options = [opt[1] for opt in search_options if opt[3]]
+                if enabled_options:
+                    selected_option = st.radio(
+                        "Search Method",
+                        options=enabled_options,
+                        horizontal=True,
+                        help="Choose your preferred research method"
+                    )
+                    
+                    # Map the selected option to the search_mode value
+                    for mode, label, _, _ in search_options:
+                        if label == selected_option:
+                            st.session_state.research_options["search_mode"] = mode
+                            break
+                else:
+                    st.warning("No search methods available. Please configure API keys.")
+
+            col1, col2 = st.columns([1, 1])
+            with col1:
+                if st.button("Apply", use_container_width=True, type="primary"):
+                    st.session_state.show_options_dialog = False
+                    st.rerun()
+            with col2:
+                if st.button("Cancel", use_container_width=True):
+                    st.session_state.show_options_dialog = False
+                    st.rerun()
+
+        # Main interface
+        st.title("ALwrity Web Researcher")
+        
+        # Primary search area with help popover
+        with st.popover("ℹ️ Keyword Research Tips"):
+            st.markdown("""
+                ### How to Get Better Results
+                1. **Primary Keywords**: Your main topic or focus
+                2. **Related Keywords**: Supporting terms that add context
+                3. **Search Depth**: Higher depth = more comprehensive but slower
+                4. **Target Audience**: Affects content recommendations
+                5. **Content Type**: Influences research focus
+                6. **Search Mode**: Choose between traditional web research(Google), AI-powered search(Tavily and Metaphor) and Deep Researcher
+            """)
+        
+        col1, col2 = st.columns([3, 1])
+        with col1:
+            st.session_state.research_options["primary_keywords"] = st.text_input(
+                "Primary Keywords", 
+                value=st.session_state.research_options["primary_keywords"],
+                placeholder="Enter main keywords for research...",
+                help="Enter your main topic or focus keywords"
+            )
+        with col2:
+            if st.button("Research Options", use_container_width=True):
+                show_research_options()
+
+        # Execute search button
+        if st.button("🔍 Start Research", type="primary", use_container_width=True):
+            if not st.session_state.research_options["primary_keywords"]:
+                st.warning("⚠️ Please enter primary keywords for research")
+                return
+            
+            try:
+                # Create compact progress display
+                progress_container = st.container()
+                with progress_container:
+                    status_col, progress_col = st.columns([3, 1])
+                    with status_col:
+                        status_display = st.empty()
+                        status_display.info("🚀 Initializing research...")
+                    with progress_col:
+                        progress_bar = st.progress(0)
+
+                # Execute search with all parameters
+                web_research_result = gpt_web_researcher(
+                    search_keywords=st.session_state.research_options["primary_keywords"],
+                    search_mode=st.session_state.research_options["search_mode"],
+                    related_keywords=st.session_state.research_options["related_keywords"],
+                    target_audience=st.session_state.research_options["target_audience"],
+                    content_type=st.session_state.research_options["content_type"],
+                    search_depth=st.session_state.research_options["search_depth"],
+                    geo_location=st.session_state.research_options["geo_location"],
+                    search_language=st.session_state.research_options["search_language"],
+                    num_results=st.session_state.research_options["num_results"],
+                    time_range=st.session_state.research_options["time_range"],
+                    include_domains=st.session_state.research_options["include_domains"],
+                    similar_url=st.session_state.research_options["similar_url"]
+                )
+                
+                if web_research_result:
+                    status_display.success("✨ Research completed!")
+                    
+                    # Display results in an organized way
+                    with st.expander("📊 Research Results", expanded=False):
+                        st.write(web_research_result)
+                else:
+                    st.warning("No results found for your search")
+                    
+            except Exception as e:
+                error_msg = f"Research failed: {str(e)}"
+                logger.error(error_msg, exc_info=True)
+                st.error(f"🚫 Research failed: {error_msg}")
+                
+    except Exception as e:
+        logger.error(f"Unexpected error in web research: {e}", exc_info=True)
+        st.error("🚫 An unexpected error occurred. Please try again.")

lib/utils/alwrity_sidebar.py

@@ -1,244 +0,0 @@
-import streamlit as st
-import logging
-
-from .config_manager import save_config
-
-# Configure logging
-logging.basicConfig(
-    level=logging.DEBUG,
-    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-    handlers=[
-        logging.StreamHandler(),  # Output to console
-        #logging.FileHandler('alwrity.log')  # Output to file
-    ]
-)
-logger = logging.getLogger(__name__)
-
-# Sidebar configuration
-def sidebar_configuration():
-    """Configure the sidebar with all necessary options."""
-    try:
-        # Configure sidebar styling
-        st.sidebar.markdown("""
-            <style>
-                [data-testid="stSidebar"] {
-                    min-width: 250px !important;
-                    max-width: 250px !important;
-                    visibility: visible !important;
-                    position: relative !important;
-                    transform: translateX(0) !important;
-                }
-                [data-testid="stSidebar"][aria-expanded="true"] {
-                    min-width: 250px !important;
-                    max-width: 250px !important;
-                    transform: translateX(0) !important;
-                }
-                [data-testid="stSidebar"][aria-expanded="false"] {
-                    min-width: 250px !important;
-                    max-width: 250px !important;
-                    transform: translateX(0) !important;
-                }
-                .stSidebar .element-container {
-                    padding: 0.5rem;
-                }
-                .stSidebar .stMarkdown {
-                    padding: 0.5rem;
-                }
-                .stSidebar .stSelectbox {
-                    padding: 0.5rem;
-                }
-                .stSidebar .stTextInput {
-                    padding: 0.5rem;
-                }
-                .stSidebar .stNumberInput {
-                    padding: 0.5rem;
-                }
-                .stSidebar .stSlider {
-                    padding: 0.5rem;
-                }
-                /* Ensure sidebar is visible */
-                section[data-testid="stSidebar"] {
-                    visibility: visible !important;
-                    transform: translateX(0) !important;
-                }
-            </style>
-        """, unsafe_allow_html=True)
-        
-        logger.info("Initializing sidebar configuration")
-        st.sidebar.title("🛠️ Personalization & Settings 🏗️")
-
-        with st.sidebar.expander("**👷 Content Personalization**"):
-            logger.debug("Setting up content personalization options")
-            blog_length = st.text_input("**Content Length (words)**", value="2000", 
-                              help="Approximate word count for blogs. Note: Actual length may vary based on GPT provider and max token count.")
-            
-            blog_tone_options = ["Casual", "Professional", "How-to", "Beginner", "Research", "Programming", "Social Media", "Customize"]
-            blog_tone = st.selectbox("**Content Tone**",
-                              options=blog_tone_options,
-                              help="Select the desired tone for the blog content.")
-            logger.debug(f"Selected blog tone: {blog_tone}")
-
-            if blog_tone == "Customize":
-                custom_tone = st.text_input("Enter the tone of your content", help="Specify the tone of your content.")
-                if custom_tone:
-                    blog_tone = custom_tone
-                    logger.debug(f"Custom tone set to: {custom_tone}")
-                else:
-                    logger.warning("Custom tone not specified")
-                    st.warning("Please specify the tone of your content.")
-            
-            blog_demographic_options = ["Professional", "Gen-Z", "Tech-savvy", "Student", "Digital Marketing", "Customize"]
-            
-            blog_demographic = st.selectbox("**Target Audience**",
-                                    options=blog_demographic_options,
-                                    help="Select the primary audience for the blog content.")
-            if blog_demographic == "Customize":
-                custom_demographic = st.text_input("Enter your target audience", 
-                                    help="Specify your target audience.",
-                                    placeholder="Eg. Domain expert, Content creator, Financial expert etc..")
-                if custom_demographic:
-                    blog_demographic = custom_demographic
-                else:
-                    st.warning("Please specify your target audience.")
-            
-            blog_type = st.selectbox("**Content Type**", 
-                              options=["Informational", "Commercial", "Company", "News", "Finance", "Competitor", "Programming", "Scholar"], 
-                              help="Select the category that best describes the blog content.")
-
-            blog_language = st.selectbox("**Content Language**", 
-                              options=["English", "Spanish", "German", "Chinese", "Arabic", "Nepali", "Hindi", "Hindustani", "Customize"], 
-                              help="Select the language in which the blog will be written.")
-            if blog_language == "Customize":
-                custom_lang = st.text_input("Enter the language of your choice", help="Specify the content language.")
-                if custom_lang:
-                    blog_language = custom_lang
-                else:
-                    st.warning("Please specify the language of your content.")
-
-            blog_output_format = st.selectbox("**Content Output Format**", 
-                              options=["markdown", "HTML", "plaintext"], 
-                              help="Select the format for the blog output.")
-
-        with st.sidebar.expander("**🩻 Images Personalization**"):
-            image_generation_model = st.selectbox("**Image Generation Model**", 
-                              options=["stable-diffusion", "dalle2", "dalle3"], 
-                              help="Select the model to generate images for the blog.")
-            number_of_blog_images = st.number_input("**Number of Blog Images**", value=1, help="Specify the number of images to include in the blog.")
-
-        with st.sidebar.expander("**🤖 LLM Personalization**"):
-            gpt_provider = st.selectbox("**GPT Provider**", 
-                              options=["google", "openai", "minstral"], 
-                              help="Select the provider for the GPT model.")
-            model = st.text_input("**Model**", value="gemini-1.5-flash-latest", help="Specify the model version to use from the selected provider.")
-            temperature = st.slider(
-                "Temperature",
-                min_value=0.1,
-                max_value=1.0,
-                value=0.7,
-                step=0.1,
-                format="%.1f",
-                help="""Temperature controls the 'creativity' or randomness of the text generated by GPT.
-                    Greater determinism with higher values indicating more randomness."""
-            )
-
-            top_p = st.slider(
-                "Top-p",
-                min_value=0.0,
-                max_value=1.0,
-                value=0.9,
-                step=0.1,
-                format="%.1f",
-                help="Top-p sampling controls the level of diversity in the generated text."
-            )
-
-            # Selectbox for max tokens
-            max_tokens_options = [500, 1000, 2000, 4000, 16000, 32000, 64000]
-            max_tokens = st.selectbox(
-                "Max Tokens",
-                options=max_tokens_options,
-                index=max_tokens_options.index(4000),
-                help="Max tokens determine the maximum length of the output sequence generated by a model."
-            )
-            n = st.number_input("N", 
-                value=1,
-                min_value=1,
-                max_value=10,
-                help="Defines the number of words or characters grouped together in a sequence when analyzing text.")
-            frequency_penalty = st.slider(
-                "Frequency Penalty",
-                min_value=0.0,
-                max_value=2.0,
-                value=1.0,
-                step=0.1,
-                format="%.1f",
-                help="Influences word selection during text generation, promoting diversity with higher values."
-            )
-
-            presence_penalty = st.slider(
-                "Presence Penalty",
-                min_value=0.0,
-                max_value=2.0,
-                value=1.0,
-                step=0.1,
-                format="%.1f",
-                help="Encourages the use of diverse words by discouraging repetition."
-            )
-
-        with st.sidebar.expander("**🕵️ Search Engine Personalization**"):
-            geographic_location = st.selectbox("**Geographic Location**", 
-                              options=["us", "in", "fr", "cn"], 
-                              help="Select the geographic location for tailoring search results.")
-            search_language = st.selectbox("**Search Language**", 
-                              options=["en", "zn-cn", "de", "hi"], 
-                              help="Select the language for the search results.")
-            number_of_results = st.number_input("**Number of Results**", 
-                        value=10,
-                        max_value=20,
-                        min_value=1,
-                        help="Specify the number of search results to retrieve.")
-            time_range = st.selectbox("**Time Range**", 
-                              options=["anytime", "past day", "past week", "past month", "past year"], 
-                              help="Select the time range for filtering search results.")
-            include_domains = st.text_input("**Include Domains**", value="", 
-                              help="List specific domains to include in search results. Leave blank to include all domains.")
-            similar_url = st.text_input("**Similar URL**", value="", help="Provide a URL to find similar results. Leave blank if not needed.")
-
-        # Storing collected inputs in a dictionary
-        config = {
-            "Blog Content Characteristics": {
-                "Blog Length": blog_length,
-                "Blog Tone": blog_tone,
-                "Blog Demographic": blog_demographic,
-                "Blog Type": blog_type,
-                "Blog Language": blog_language,
-                "Blog Output Format": blog_output_format
-            },
-            "Blog Images Details": {
-                "Image Generation Model": image_generation_model,
-                "Number of Blog Images": number_of_blog_images
-            },
-            "LLM Options": {
-                "GPT Provider": gpt_provider,
-                "Model": model,
-                "Temperature": temperature,
-                "Top-p": top_p,
-                "Max Tokens": max_tokens,
-                "N": n,
-                "Frequency Penalty": frequency_penalty,
-                "Presence Penalty": presence_penalty
-            },
-            "Search Engine Parameters": {
-                "Geographic Location": geographic_location,
-                "Search Language": search_language,
-                "Number of Results": number_of_results,
-                "Time Range": time_range,
-                "Include Domains": include_domains,
-                "Similar URL": similar_url
-            }
-        }
-
-        # Writing the configuration to a file whenever a change is made
-        save_config(config)
-    except Exception as e:
-        logger.error(f"Error configuring sidebar: {str(e)}")
-        st.error(f"Error configuring sidebar: {str(e)}")

lib/utils/alwrity_utils.py

@@ -457,26 +457,6 @@ def competitor_analysis():
             st.error("Please enter a valid URL.")
 
 
-def do_web_research():
-    """ Input keywords and do web research and present a report."""
-    st.title("Web Research Assistant")
-    st.write("Enter keywords for web research. The keywords should be at least three words long.")
-    
-    search_keywords = st.text_input("Search Keywords", placeholder="Enter keywords for web research...")
-    if st.button("Start Web Research"):
-        if search_keywords and len(search_keywords.split()) >= 3:
-            try:
-                st.info(f"Starting web research on given keywords: {search_keywords}")
-                with st.spinner("Performing web research..."):
-                    web_research_result = gpt_web_researcher(search_keywords)
-                st.success("Web research completed successfully!")
-                st.write(web_research_result)
-            except Exception as err:
-                st.error(f"ERROR: Failed to do web research: {err}")
-        else:
-            st.warning("Search keywords should be at least three words long. Please try again.")
-
-
 def ai_finance_ta_writer():
     st.markdown("<div class='sub-header'>AI Financial Technical Analysis Writer</div>", unsafe_allow_html=True)
 

lib/utils/api_key_manager/components/personalization_setup.py

@@ -8,11 +8,13 @@ from typing import Dict, Any
 from ..manager import APIKeyManager
 from ....web_crawlers.async_web_crawler import AsyncWebCrawlerService
 from ....personalization.style_analyzer import StyleAnalyzer
-from pages.style_utils import (
-    get_analysis_section,
+from lib.utils.style_utils import (
+    get_test_config_styles,
     get_glass_container,
     get_info_section,
-    get_example_box
+    get_example_box,
+    get_analysis_section,
+    get_style_guide_html
 )
 from .base import render_navigation_buttons
 from .alwrity_integrations import render_alwrity_integrations
@@ -618,7 +620,7 @@ def render_personalization_setup(api_key_manager: APIKeyManager) -> Dict[str, An
                     st.warning("Please provide either a website URL or content samples")
         
         with col2:
-            st.markdown("""
+            st.markdown(get_glass_container("""
                 ### How ALwrity Discovers Your Style
                 
                 **AI-Powered Style Analysis**
@@ -651,10 +653,15 @@ def render_personalization_setup(api_key_manager: APIKeyManager) -> Dict[str, An
                 - Maintain consistency across all content
                 - Optimize for your target audience
                 - Ensure brand voice alignment
-            """)
+            """))
             
             # API Configuration Form
-            st.markdown("### API Configuration")
+            st.markdown(get_glass_container("""
+                ### API Configuration
+                
+                Configure your API settings for optimal content generation.
+            """))
+            
             with st.form("ai_config_form"):
                 # API Keys
                 st.text_input("OpenAI API Key", type="password", key="openai_key")

lib/utils/content_generators.py

@@ -1,12 +1,12 @@
 import streamlit as st
 from lib.utils.alwrity_utils import (
     blog_from_keyword, ai_agents_team, essay_writer, ai_news_writer,
-    ai_finance_ta_writer, ai_social_writer, do_web_research, competitor_analysis
+    ai_finance_ta_writer, competitor_analysis
 )
+from lib.alwrity_ui.keyword_web_researcher import do_web_research
 from lib.ai_writers.ai_story_writer.story_writer import story_input_section
 from lib.ai_writers.ai_product_description_writer import write_ai_prod_desc
 #from lib.content_planning_calender.content_planning_agents_alwrity_crew import ai_agents_content_planner
-from lib.utils.seo_tools import ai_seo_tools
 
 
 def ai_writers():
@@ -39,30 +39,67 @@ def ai_writers():
 
 
 def content_planning_tools():
-    st.markdown("""**Alwrity content Ideation & Planning** : Provide few keywords to do comprehensive web research.
-             Provide few keywords to get Google, Neural, pytrends analysis. Know keywords, blog titles to target.
-             Generate months long content calendar around given keywords.""")
+    # Add custom CSS for compact layout
+    st.markdown("""
+        <style>
+            /* Reduce top padding of main container */
+            .main .block-container {
+                padding-top: 0rem !important;
+                padding-bottom: 1rem !important;
+            }
+            
+            /* Reduce spacing between elements */
+            .stTabs {
+                margin-top: 0.5rem !important;
+            }
+            
+            /* Make markdown text more compact */
+            .element-container {
+                margin-bottom: 0.5rem !important;
+            }
+            
+            /* Adjust subheader margins */
+            .stMarkdown h3 {
+                margin-top: 0 !important;
+                margin-bottom: 0.5rem !important;
+            }
+        </style>
+    """, unsafe_allow_html=True)
     
-    options = [
-        "Keywords Researcher",
-        "Competitor Analysis",
-        "Content Calender Ideator"
-    ]
-    choice = st.radio("Select a content planning tool:", options, index=0, format_func=lambda x: f"🔍 {x}")
+    # Make description more compact using a smaller font
+    st.markdown("""
+        <div style='font-size: 0.9em; margin-bottom: 0.5rem;'>
+            <strong>Alwrity content Ideation & Planning</strong>: Provide few keywords to do comprehensive web research.
+            Provide few keywords to get Google, Neural, pytrends analysis. Know keywords, blog titles to target.
+            Generate months long content calendar around given keywords.
+        </div>
+    """, unsafe_allow_html=True)
     
-    if choice == "Keywords Researcher":
+    # Create tabs with reduced spacing
+    tab_keywords, tab_competitor, tab_calendar = st.tabs([
+        "🔍 Keywords Researcher",
+        "📊 Competitor Analysis",
+        "📅 Content Calendar Ideator"
+    ])
+    
+    # Keywords Researcher tab
+    with tab_keywords:
         do_web_research()
-    elif choice == "Competitor Analysis":
+        
+    # Competitor Analysis tab
+    with tab_competitor:
         competitor_analysis()
-    elif choice == "Content Calender Ideator":
+        
+    # Content Calendar Ideator tab
+    with tab_calendar:
         plan_keywords = st.text_input(
             "**Enter Your main Keywords to get 2 months content calendar:**",
             placeholder="Enter 2-3 main keywords to generate AI content calendar with keyword researched blog titles",
             help="The keywords are the ones where you would want to generate 50-60 blogs/articles on."
         )
-        if st.button("**Ideate Content Calender**"):
+        if st.button("**Ideate Content Calendar**"):
             if plan_keywords:
                 #ai_agents_content_planner(plan_keywords)
-                st.header("COming Soon.")
+                st.header("Coming Soon.")
             else:
                 st.error("Come on, really, Enter some keywords to plan on..")

lib/utils/settings_page.py

@@ -0,0 +1,438 @@
+import streamlit as st
+from loguru import logger
+import asyncio
+from lib.web_crawlers.async_web_crawler import AsyncWebCrawlerService
+from lib.personalization.style_analyzer import StyleAnalyzer
+import sys
+
+# Configure logger
+logger.remove()  # Remove default handler
+logger.add(
+    "logs/settings_page.log",
+    rotation="500 MB",
+    retention="10 days",
+    level="DEBUG",
+    format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {message}",
+    backtrace=True,
+    diagnose=True
+)
+logger.add(
+    sys.stdout,
+    level="INFO",
+    format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{message}</cyan>"
+)
+
+def display_style_analysis(analysis_results: dict):
+    """Display the style analysis results in a structured format."""
+    try:
+        # Writing Style Section
+        st.markdown("### 🎨 Writing Style Analysis")
+        writing_style = analysis_results.get("writing_style", {})
+        writing_style_content = f"""
+            <ul>
+                <li><strong>Tone:</strong> {writing_style.get("tone", "N/A")}</li>
+                <li><strong>Voice:</strong> {writing_style.get("voice", "N/A")}</li>
+                <li><strong>Complexity:</strong> {writing_style.get("complexity", "N/A")}</li>
+                <li><strong>Engagement Level:</strong> {writing_style.get("engagement_level", "N/A")}</li>
+            </ul>
+        """
+        st.markdown(writing_style_content, unsafe_allow_html=True)
+        
+        # Content Characteristics Section
+        content_chars = analysis_results.get("content_characteristics", {})
+        content_chars_content = f"""
+            <ul>
+                <li><strong>Sentence Structure:</strong> {content_chars.get("sentence_structure", "N/A")}</li>
+                <li><strong>Vocabulary Level:</strong> {content_chars.get("vocabulary_level", "N/A")}</li>
+                <li><strong>Paragraph Organization:</strong> {content_chars.get("paragraph_organization", "N/A")}</li>
+                <li><strong>Content Flow:</strong> {content_chars.get("content_flow", "N/A")}</li>
+            </ul>
+        """
+        st.markdown(content_chars_content, unsafe_allow_html=True)
+        
+        # Target Audience Section
+        target_audience = analysis_results.get("target_audience", {})
+        target_audience_content = f"""
+            <ul>
+                <li><strong>Demographics:</strong> {', '.join(target_audience.get("demographics", ["N/A"]))}</li>
+                <li><strong>Expertise Level:</strong> {target_audience.get("expertise_level", "N/A")}</li>
+                <li><strong>Industry Focus:</strong> {target_audience.get("industry_focus", "N/A")}</li>
+                <li><strong>Geographic Focus:</strong> {target_audience.get("geographic_focus", "N/A")}</li>
+            </ul>
+        """
+        st.markdown(target_audience_content, unsafe_allow_html=True)
+        
+        # Content Type Section
+        content_type = analysis_results.get("content_type", {})
+        content_type_content = f"""
+            <ul>
+                <li><strong>Primary Type:</strong> {content_type.get("primary_type", "N/A")}</li>
+                <li><strong>Secondary Types:</strong> {', '.join(content_type.get("secondary_types", ["N/A"]))}</li>
+                <li><strong>Purpose:</strong> {content_type.get("purpose", "N/A")}</li>
+                <li><strong>Call to Action:</strong> {content_type.get("call_to_action", "N/A")}</li>
+            </ul>
+        """
+        st.markdown(content_type_content, unsafe_allow_html=True)
+        
+        # Recommended Settings Section
+        recommended = analysis_results.get("recommended_settings", {})
+        recommended_content = f"""
+            <ul>
+                <li><strong>Writing Tone:</strong> {recommended.get("writing_tone", "N/A")}</li>
+                <li><strong>Target Audience:</strong> {recommended.get("target_audience", "N/A")}</li>
+                <li><strong>Content Type:</strong> {recommended.get("content_type", "N/A")}</li>
+                <li><strong>Creativity Level:</strong> {recommended.get("creativity_level", "N/A")}</li>
+                <li><strong>Geographic Location:</strong> {recommended.get("geographic_location", "N/A")}</li>
+            </ul>
+        """
+        st.markdown(recommended_content, unsafe_allow_html=True)
+        
+    except Exception as e:
+        logger.error(f"Error displaying style analysis: {str(e)}")
+        st.error(f"Error displaying analysis results: {str(e)}")
+
+def render_settings_page():
+    """Renders the settings page with all configuration options in tabs"""
+    st.title("🛠️ Settings & Configuration")
+    
+    # Create tabs for different settings categories
+    tabs = st.tabs([
+        "👷 Content",
+        "🩻 Images",
+        "🤖 LLM",
+        "🕵️ Search",
+        "🎨 AI Personalization"
+    ])
+    
+    # Content Settings Tab
+    with tabs[0]:
+        st.header("Content Personalization")
+        blog_length = st.text_input(
+            "**Content Length (words)**",
+            value="2000",
+            key="settings_blog_length",
+            help="Approximate word count for blogs. Note: Actual length may vary based on GPT provider and max token count."
+        )
+
+        blog_tone_options = ["Casual", "Professional", "How-to", "Beginner", "Research", "Programming", "Social Media", "Customize"]
+        blog_tone = st.selectbox(
+            "**Content Tone**",
+            options=blog_tone_options,
+            key="settings_blog_tone",
+            help="Select the desired tone for the blog content."
+        )
+
+        if blog_tone == "Customize":
+            custom_tone = st.text_input(
+                "Enter the tone of your content",
+                key="settings_custom_tone",
+                help="Specify the tone of your content."
+            )
+            if custom_tone:
+                blog_tone = custom_tone
+            else:
+                st.warning("Please specify the tone of your content.")
+
+        blog_demographic_options = ["Professional", "Gen-Z", "Tech-savvy", "Student", "Digital Marketing", "Customize"]
+        blog_demographic = st.selectbox(
+            "**Target Audience**",
+            options=blog_demographic_options,
+            key="settings_blog_demographic",
+            help="Select the primary audience for the blog content."
+        )
+
+        blog_type = st.selectbox(
+            "**Content Type**",
+            options=["Informational", "Commercial", "Company", "News", "Finance", "Competitor", "Programming", "Scholar"],
+            key="settings_blog_type",
+            help="Select the category that best describes the blog content."
+        )
+
+        blog_language = st.selectbox(
+            "**Content Language**",
+            options=["English", "Spanish", "German", "Chinese", "Arabic", "Nepali", "Hindi", "Hindustani", "Customize"],
+            key="settings_blog_language",
+            help="Select the language in which the blog will be written."
+        )
+
+        blog_output_format = st.selectbox(
+            "**Content Output Format**",
+            options=["markdown", "HTML", "plaintext"],
+            key="settings_blog_output_format",
+            help="Select the format for the blog output."
+        )
+
+    # Images Settings Tab
+    with tabs[1]:
+        st.header("Images Personalization")
+        image_generation_model = st.selectbox(
+            "**Image Generation Model**",
+            options=["stable-diffusion", "dalle2", "dalle3"],
+            key="settings_image_model",
+            help="Select the model to generate images for the blog."
+        )
+        
+        number_of_blog_images = st.number_input(
+            "**Number of Blog Images**",
+            value=1,
+            min_value=1,
+            max_value=10,
+            key="settings_number_of_images",
+            help="Specify the number of images to include in the blog."
+        )
+
+    # LLM Settings Tab
+    with tabs[2]:
+        st.header("LLM Personalization")
+        gpt_provider = st.selectbox(
+            "**GPT Provider**",
+            options=["google", "openai", "minstral"],
+            key="settings_gpt_provider",
+            help="Select the provider for the GPT model."
+        )
+
+        model = st.text_input(
+            "**Model**",
+            value="gemini-1.5-flash-latest",
+            key="settings_model",
+            help="Specify the model version to use from the selected provider."
+        )
+
+        col1, col2 = st.columns(2)
+        with col1:
+            temperature = st.slider(
+                "Temperature",
+                min_value=0.1,
+                max_value=1.0,
+                value=0.7,
+                step=0.1,
+                key="settings_temperature",
+                help="Controls the creativity level of the generated text."
+            )
+            
+            max_tokens = st.selectbox(
+                "Max Tokens",
+                options=[500, 1000, 2000, 4000, 16000, 32000, 64000],
+                index=3,
+                key="settings_max_tokens",
+                help="Maximum length of the output sequence."
+            )
+
+        with col2:
+            top_p = st.slider(
+                "Top-p",
+                min_value=0.0,
+                max_value=1.0,
+                value=0.9,
+                step=0.1,
+                key="settings_top_p",
+                help="Controls diversity in text generation."
+            )
+            
+            frequency_penalty = st.slider(
+                "Frequency Penalty",
+                min_value=0.0,
+                max_value=2.0,
+                value=1.0,
+                step=0.1,
+                key="settings_frequency_penalty",
+                help="Reduces word repetition in output."
+            )
+
+    # Search Settings Tab
+    with tabs[3]:
+        st.header("Search Engine Personalization")
+        geographic_location = st.selectbox(
+            "**Geographic Location**",
+            options=["us", "in", "fr", "cn"],
+            key="settings_geographic_location",
+            help="Select the geographic location for tailoring search results."
+        )
+
+        search_language = st.selectbox(
+            "**Search Language**",
+            options=["en", "zn-cn", "de", "hi"],
+            key="settings_search_language",
+            help="Select the language for the search results."
+        )
+
+        number_of_results = st.number_input(
+            "**Number of Results**",
+            value=10,
+            min_value=1,
+            max_value=20,
+            key="settings_number_of_results",
+            help="Specify the number of search results to retrieve."
+        )
+
+        time_range = st.selectbox(
+            "**Time Range**",
+            options=["anytime", "past day", "past week", "past month", "past year"],
+            key="settings_time_range",
+            help="Select the time range for filtering search results."
+        )
+
+        include_domains = st.text_input(
+            "**Include Domains**",
+            value="",
+            key="settings_include_domains",
+            help="List specific domains to include in search results (comma-separated)."
+        )
+
+        similar_url = st.text_input(
+            "**Similar URL**",
+            value="",
+            key="settings_similar_url",
+            help="Provide a URL to find similar results."
+        )
+
+    # AI Personalization Tab
+    with tabs[4]:
+        st.header("🎨 AI Style Analysis")
+        st.markdown("""
+            <div style='background-color: rgba(255, 255, 255, 0.1); padding: 20px; border-radius: 10px; margin-bottom: 20px;'>
+                <p>Enter a website URL or provide content samples to analyze your writing style and get personalized recommendations.</p>
+            </div>
+        """, unsafe_allow_html=True)
+        
+        # Create two columns for the layout
+        col1, col2 = st.columns([2, 1])
+        
+        with col1:
+            # Website URL input
+            st.markdown("### Website URL")
+            url = st.text_input(
+                "Enter your website URL",
+                placeholder="https://example.com",
+                key="settings_website_url",
+                help="Provide your website URL to analyze your content style. Leave empty if you want to provide written samples instead."
+            )
+            
+            # Alternative: Written samples
+            if not url:
+                st.markdown("### Written Samples")
+                st.markdown("""
+                    <div style='background-color: rgba(255, 255, 255, 0.1); padding: 20px; border-radius: 10px; margin-bottom: 20px;'>
+                        <p>No website URL? No problem! You can provide written samples of your content instead.</p>
+                        <p>Share your best articles, blog posts, or any content that represents your writing style.</p>
+                    </div>
+                """, unsafe_allow_html=True)
+                samples = st.text_area(
+                    "Paste your content samples here",
+                    key="settings_content_samples",
+                    help="Paste 2-3 samples of your best content. This helps ALwrity understand your writing style."
+                )
+            
+            # ALwrity Style button
+            st.markdown("<div style='height: 20px'></div>", unsafe_allow_html=True)
+            if st.button("🎨 Analyze Style", use_container_width=True, key="settings_analyze_style"):
+                if url:
+                    with st.status("Starting style analysis...", expanded=True) as status:
+                        try:
+                            # Step 1: Initialize crawler
+                            status.update(label="Step 1/4: Initializing web crawler...", state="running")
+                            crawler_service = AsyncWebCrawlerService()
+                            
+                            # Step 2: Crawl website
+                            status.update(label="Step 2/4: Crawling website content...", state="running")
+                            loop = asyncio.new_event_loop()
+                            asyncio.set_event_loop(loop)
+                            result = loop.run_until_complete(crawler_service.crawl_website(url))
+                            loop.close()
+                            
+                            if result.get('success', False):
+                                content = result.get('content', {})
+                                
+                                # Step 3: Initialize style analyzer
+                                status.update(label="Step 3/4: Analyzing content style...", state="running")
+                                style_analyzer = StyleAnalyzer()
+                                
+                                # Step 4: Perform style analysis
+                                status.update(label="Step 4/4: Generating style recommendations...", state="running")
+                                style_analysis = style_analyzer.analyze_content_style(content)
+                                
+                                if style_analysis.get('error'):
+                                    status.update(label="Analysis failed", state="error")
+                                    st.error(f"Style analysis failed: {style_analysis['error']}")
+                                else:
+                                    status.update(label="Analysis complete!", state="complete")
+                                    # Display style analysis results
+                                    display_style_analysis(style_analysis)
+                                    
+                                    # Display original content in tabs
+                                    tab1, tab2, tab3 = st.tabs(["Content", "Metadata", "Links"])
+                                    
+                                    with tab1:
+                                        st.markdown("### Main Content")
+                                        st.markdown(content.get('main_content', 'No content found'))
+                                        
+                                    with tab2:
+                                        st.markdown("### Metadata")
+                                        st.markdown(f"""
+                                            **Title:** {content.get('title', 'No title found')}
+                                            
+                                            **Description:** {content.get('description', 'No description found')}
+                                            
+                                            **Meta Tags:**
+                                            {content.get('meta_tags', {})}
+                                        """)
+                                        
+                                    with tab3:
+                                        st.markdown("### Links")
+                                        for link in content.get('links', []):
+                                            st.markdown(f"- [{link.get('text', '')}]({link.get('href', '')})")
+                            else:
+                                status.update(label="Crawling failed", state="error")
+                                st.error("Failed to crawl the website. Please check the URL and try again.")
+                        except Exception as e:
+                            status.update(label="Analysis failed", state="error")
+                            st.error(f"An error occurred during analysis: {str(e)}")
+                elif samples:
+                    with st.status("Starting style analysis...", expanded=True) as status:
+                        try:
+                            # Initialize style analyzer
+                            status.update(label="Analyzing content style...", state="running")
+                            style_analyzer = StyleAnalyzer()
+                            
+                            # Perform style analysis
+                            style_analysis = style_analyzer.analyze_content_style({"main_content": samples})
+                            
+                            if style_analysis.get('error'):
+                                status.update(label="Analysis failed", state="error")
+                                st.error(f"Style analysis failed: {style_analysis['error']}")
+                            else:
+                                status.update(label="Analysis complete!", state="complete")
+                                # Display style analysis results
+                                display_style_analysis(style_analysis)
+                        except Exception as e:
+                            status.update(label="Analysis failed", state="error")
+                            st.error(f"An error occurred during analysis: {str(e)}")
+                else:
+                    st.warning("Please provide either a website URL or content samples to analyze.")
+
+    # Save Settings Button
+    if st.button("💾 Save Settings", type="primary", use_container_width=True, key="settings_save_button"):
+        # Save all settings to session state
+        st.session_state.update({
+            'blog_length': blog_length,
+            'blog_tone': blog_tone,
+            'blog_demographic': blog_demographic,
+            'blog_type': blog_type,
+            'blog_language': blog_language,
+            'blog_output_format': blog_output_format,
+            'image_generation_model': image_generation_model,
+            'number_of_blog_images': number_of_blog_images,
+            'gpt_provider': gpt_provider,
+            'model': model,
+            'temperature': temperature,
+            'top_p': top_p,
+            'max_tokens': max_tokens,
+            'frequency_penalty': frequency_penalty,
+            'geographic_location': geographic_location,
+            'search_language': search_language,
+            'number_of_results': number_of_results,
+            'time_range': time_range,
+            'include_domains': include_domains,
+            'similar_url': similar_url
+        })
+        st.success("✅ Settings saved successfully!")

lib/utils/style_utils.py

@@ -267,79 +267,147 @@ def get_glassmorphic_styles() -> str:
     </style>
     """
 
+def get_test_config_styles():
+    """Returns CSS styles for the test configuration page."""
+    return """
+        <style>
+            .stApp {
+                background: linear-gradient(135deg, #f5f7fa 0%, #c3cfe2 100%);
+            }
+            
+            .stButton > button {
+                background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+                color: white;
+                border: none;
+                padding: 0.5rem 1rem;
+                border-radius: 8px;
+                font-weight: 500;
+                transition: all 0.3s ease;
+            }
+            
+            .stButton > button:hover {
+                transform: translateY(-2px);
+                box-shadow: 0 4px 12px rgba(0,0,0,0.1);
+            }
+            
+            .stTextInput > div > div > input {
+                border-radius: 8px;
+                border: 1px solid rgba(0,0,0,0.1);
+                padding: 0.5rem 1rem;
+            }
+            
+            .stSelectbox > div > div > select {
+                border-radius: 8px;
+                border: 1px solid rgba(0,0,0,0.1);
+                padding: 0.5rem 1rem;
+            }
+            
+            .stTextArea > div > div > textarea {
+                border-radius: 8px;
+                border: 1px solid rgba(0,0,0,0.1);
+                padding: 0.5rem 1rem;
+            }
+            
+            .stMarkdown {
+                font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+            }
+            
+            h1, h2, h3 {
+                color: #2c3e50;
+                font-weight: 600;
+            }
+            
+            .stSuccess {
+                background: linear-gradient(135deg, #43c6ac 0%, #191654 100%);
+                padding: 1rem;
+                border-radius: 8px;
+                color: white;
+            }
+            
+            .stError {
+                background: linear-gradient(135deg, #ff6b6b 0%, #ff8e8e 100%);
+                padding: 1rem;
+                border-radius: 8px;
+                color: white;
+            }
+            
+            .stWarning {
+                background: linear-gradient(135deg, #ffd700 0%, #ffa500 100%);
+                padding: 1rem;
+                border-radius: 8px;
+                color: #2c3e50;
+            }
+        </style>
+    """
+
 def get_glass_container(content: str) -> str:
-    """Wrap content in a glass container."""
+    """Returns HTML for a glass-morphism container."""
     return f"""
-        <div class="glass-container">
+        <div style='
+            background: rgba(255, 255, 255, 0.1);
+            backdrop-filter: blur(10px);
+            border-radius: 10px;
+            padding: 20px;
+            margin: 20px 0;
+            box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+        '>
             {content}
         </div>
     """
 
 def get_info_section(content: str) -> str:
-    """Wrap content in an info section."""
+    """Returns HTML for an info section."""
     return f"""
-        <div class="info-section">
+        <div style='
+            background: rgba(255, 255, 255, 0.1);
+            border-radius: 10px;
+            padding: 20px;
+            margin: 20px 0;
+        '>
             {content}
         </div>
     """
 
 def get_example_box(content: str) -> str:
-    """Wrap content in an example box."""
+    """Returns HTML for an example box."""
     return f"""
-        <div class="example-box">
+        <div style='
+            background: rgba(255, 255, 255, 0.1);
+            border-radius: 10px;
+            padding: 20px;
+            margin: 20px 0;
+            border-left: 4px solid #667eea;
+        '>
             {content}
         </div>
     """
 
 def get_analysis_section(title: str, content: str) -> str:
-    """Create an analysis section with title and content."""
+    """Returns HTML for an analysis section."""
     return f"""
-        <div class="analysis-section">
+        <div style='
+            background: rgba(255, 255, 255, 0.1);
+            border-radius: 10px;
+            padding: 20px;
+            margin: 20px 0;
+        '>
             <h3>{title}</h3>
             {content}
         </div>
     """
 
 def get_style_guide_html() -> str:
-    """
-    Get the style guide HTML content.
-    
-    Returns:
-        str: HTML content for the style guide section
-    """
+    """Returns HTML for the style guide section."""
     return """
-        ### How ALwrity Discovers Your Style
-        
-        **AI-Powered Style Analysis**
-        
-        ALwrity AI analyzes your existing content to understand your unique writing style and preferences. This helps us generate content that matches your voice perfectly.
-        
-        **Step 1: Content Analysis**
-        
-        We'll analyze your website content or written samples to understand:
-        
-        - Writing tone and voice
-        - Vocabulary and language style
-        - Content structure and formatting
-        - Target audience and engagement style
-        
-        **Step 2: Style Recommendations**
-        
-        Based on the analysis, we'll provide:
-        
-        - Personalized writing guidelines
-        - Content structure templates
-        - Tone and voice recommendations
-        - Audience engagement strategies
-        
-        **Step 3: Content Generation**
-        
-        Finally, we'll use these insights to:
-        
-        - Generate content that matches your style
-        - Maintain consistency across all content
-        - Optimize for your target audience
-        - Ensure brand voice alignment
+        <div style='
+            background: rgba(255, 255, 255, 0.1);
+            border-radius: 10px;
+            padding: 20px;
+            margin: 20px 0;
+        '>
+            <h3>Style Guide</h3>
+            <p>This section will contain your style guide and brand guidelines.</p>
+        </div>
     """
 
 def get_test_config_styles() -> str:

lib/utils/test_config_settings.py

@@ -148,8 +148,8 @@ def render_test_config_settings():
                 # Set session state for navigation
                 st.session_state.current_step = 4
                 st.session_state.next_step = "personalization_setup"
-                # Navigate back to personalization setup
-                st.switch_page("pages/personalization_setup.py")
+                # Navigate back to the main page where personalization setup is rendered
+                st.switch_page("alwrity.py")
         
         # Title and description
         st.title("🎨 Find Your Style with ALwrity")

lib/utils/ui_setup.py

@@ -4,6 +4,7 @@ from lib.utils.file_processor import load_image
 from lib.utils.content_generators import content_planning_tools, ai_writers
 from lib.utils.alwrity_utils import ai_social_writer
 from lib.utils.seo_tools import ai_seo_tools
+from lib.utils.settings_page import render_settings_page
 
 
 def setup_ui():
@@ -67,40 +68,73 @@ def setup_ui():
                 border-radius: 8px;
                 color: white;
             }
+
+            /* Sidebar navigation styling */
+            .sidebar-nav {
+                padding: 1rem 0;
+            }
+            
+            .nav-button {
+                width: 100%;
+                text-align: left;
+                padding: 0.5rem 1rem;
+                background: transparent;
+                border: none;
+                color: #2c3e50;
+                font-weight: 500;
+                cursor: pointer;
+                transition: all 0.3s ease;
+                margin: 0.2rem 0;
+                border-radius: 4px;
+            }
+            
+            .nav-button:hover {
+                background: rgba(0,0,0,0.05);
+                padding-left: 1.5rem;
+            }
+            
+            .nav-button.active {
+                background: #1565C0;
+                color: white;
+            }
         </style>
     """, unsafe_allow_html=True)
 
-    image_base64 = load_image("lib/workspace/alwrity_logo.png")
-    st.markdown(f"""
-    <div class='main-header'>
-        <img src='data:image/png;base64,{image_base64}' alt='Alwrity Logo' style='height: 50px; margin-right: 10px; vertical-align: middle;'>
-        Welcome to Alwrity!
-    </div>
-    """, unsafe_allow_html=True)
 
+def setup_alwrity_ui():
+    """Sets up the main navigation in the sidebar."""
+    # Initialize session state for active tab if not exists
+    if 'active_tab' not in st.session_state:
+        st.session_state.active_tab = "Content Planning"
 
-def setup_tabs():
-    """Sets up the main tabs in the Streamlit app."""
-    tab1, tab2, tab3, tab4, tab5, tab6 = st.tabs(
-        ["📅Content Planning", " 📝🤖AI Writers", "🤝🤖Agents Teams", "🛠️🔍AI SEO tools", "📱AI Social Tools", " 💬Ask Alwrity"])
-    with tab1:
-        content_planning_tools()
+    # Define the navigation items with their icons and functions
+    nav_items = {
+        "Content Planning": ("📅", content_planning_tools),
+        "AI Writers": ("📝", ai_writers),
+        "Agents Teams": ("🤝", lambda: st.subheader("Agents Teams - Coming Soon!")),
+        "AI SEO Tools": ("🔍", ai_seo_tools),
+        "AI Social Tools": ("📱", ai_social_writer),
+        "Ask Alwrity": ("💬", lambda: (
+            st.subheader("Chat with your Data, Chat with any Data.. COMING SOON !"),
+            st.markdown("Create a collection by uploading files (PDF, MD, CSV, etc), or crawl a data source (Websites, more sources coming soon."),
+            st.markdown("One can ask/chat, summarize and do semantic search over the uploaded data")
+        )),
+        "ALwrity Settings": ("⚙️", render_settings_page)
+    }
 
-    with tab2:
-        ai_writers()
+    # Create sidebar navigation
+    st.sidebar.markdown("### ALwrity Options")
+    st.sidebar.markdown('<div class="sidebar-nav">', unsafe_allow_html=True)
 
-    with tab3:
-        #ai_agents_team()
-        st.subheader("Agents Teams")
-        
-    with tab4:
-        ai_seo_tools()
+    # Create navigation buttons
+    for name, (icon, func) in nav_items.items():
+        button_class = "nav-button active" if st.session_state.active_tab == name else "nav-button"
+        if st.sidebar.button(f"{icon} {name}", key=f"nav_{name}", 
+                           help=f"Navigate to {name}", use_container_width=True):
+            st.session_state.active_tab = name
 
-    with tab5:
-        ai_social_writer()
+    st.sidebar.markdown('</div>', unsafe_allow_html=True)
 
-    with tab6:
-        st.subheader("Chat with your Data, Chat with any Data.. COMING SOON !")
-        st.markdown("Create a collection by uploading files (PDF, MD, CSV, etc), or crawl a data source (Websites, more sources coming soon.")
-        st.markdown("One can ask/chat, summarize and do semantic search over the uploaded data")
-        # alwrity_chat_docqa()
+    # Display content based on active tab
+    st.title(f"{nav_items[st.session_state.active_tab][0]} {st.session_state.active_tab}")
+    nav_items[st.session_state.active_tab][1]()

lib/workspace/alwrity_ui_styling.css

@@ -13,13 +13,13 @@ body {
 
 /* Main header styling */
 .main-header {
-    font-size: 2.5em;
+    font-size: 2em;
     font-weight: bold;
     color: #1565C0;
-    margin-bottom: 20px;
+    margin-bottom: 2px;
     text-align: center;
     text-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);
-    padding-top: 10px;
+    padding-top: 1px;
 }
 
 /* Sub-header styling */
@@ -32,49 +32,355 @@ body {
     text-align: center;
 }
 
-/* Navigation tabs styling */
-.stTabs [role="tab"] {
-    font-size: 18px;
-    font-weight: bold;
-    color: white;
-    background: #1565C0;
-    padding: 6px 10px;
-    margin: 5px;
+/* Enhanced Tab styling with dark red gradients */
+.stTabs {
+    margin-top: 0.5rem;
+    background: white;
+    padding: 0.5rem;
+    border-radius: 2px;
+    box-shadow: 0 2px 2px rgba(0, 0, 0, 0.05);
+}
+
+.stTabs [data-baseweb="tab-list"] {
+    gap: 8px;
+    background: linear-gradient(135deg, #f8fafc, #f1f5f9);
+    padding: 8px;
     border-radius: 8px;
-    border: 2px solid #ddd;
-    transition: background 0.3s ease, padding 0.3s ease;
 }
 
-.stTabs [role="tab"]:hover {
-    background: #1976D2;
+.stTabs [data-baseweb="tab"] {
+    height: auto;
+    padding: 12px 20px;
+    color: #E2E8F0;
+    border-radius: 8px;
+    font-weight: 600;
+    font-size: 15px;
+    background: linear-gradient(135deg, #4A5568, #2D3748);
+    border: 1px solid rgba(255, 255, 255, 0.1);
+    transition: all 0.3s ease;
+    letter-spacing: 0.3px;
+    background: white;
+    border-radius: 6px;
+    padding: 8px 16px;
+    font-weight: 600;
+    color: #475569;
+    transition: all 0.2s ease;
 }
 
-.stTabs [role="tab"][aria-selected="true"] {
-    background: #E3F2FD;
-    color: #333;
-    border: 2px solid #1565C0;
-    font-weight: bold;
+.stTabs [data-baseweb="tab"]:hover {
+    color: #FFFFFF;
+    background: linear-gradient(135deg, #822727, #991B1B);
+    border-color: rgba(255, 255, 255, 0.2);
+    transform: translateY(-1px);
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.2);
+    background: #f1f5f9;
+    color: #1e293b;
 }
 
-/* Sidebar header styling */
-.sidebar-header {
-    font-size: 1.5em;
-    font-weight: bold;
-    color: #333;
-    margin-bottom: 20px;
+.stTabs [data-baseweb="tab"][aria-selected="true"] {
+    color: #FFFFFF;
+    background: linear-gradient(135deg, #3182CE, #2C5282);
+    border-color: #DC2626;
+    box-shadow: 0 4px 12px rgba(220, 38, 38, 0.3);
+    position: relative;
+    background: linear-gradient(135deg, #3182ce, #2c5282);
+    color: white;
 }
 
-/* Sidebar option styling */
-.sidebar-option {
-    margin-bottom: 10px;
-    font-size: 1.5em;
-    color: #1565C0;
+.stTabs [data-baseweb="tab"][aria-selected="true"]::after {
+    content: '';
+    position: absolute;
+    bottom: -2px;
+    left: 10%;
+    width: 80%;
+    height: 2px;
+    background: linear-gradient(90deg, transparent, #FFFFFF, transparent);
+    border-radius: 2px;
+}
+
+.stTabs [data-baseweb="tab-panel"] {
+    padding: 20px;
+    background: linear-gradient(135deg, #FFFFFF, #F8FAFC);
+    border-radius: 12px;
+    margin-top: 10px;
+    border: 1px solid rgba(226, 232, 240, 0.8);
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.05);
+}
+
+/* Enhanced tab content for better readability */
+.stTabs [data-baseweb="tab-panel"] p {
+    color: #1A202C;
+    line-height: 1.7;
+    font-size: 15px;
+}
+
+.stTabs [data-baseweb="tab-panel"] ul {
+    margin-top: 1rem;
+    margin-bottom: 1rem;
+    padding-left: 1.5rem;
+}
+
+.stTabs [data-baseweb="tab-panel"] li {
+    color: #2D3748;
+    margin-bottom: 0.5rem;
+    line-height: 1.6;
+}
+
+/* Tab content headings */
+.stTabs [data-baseweb="tab-panel"] strong {
+    color: #1A202C;
+    font-weight: 600;
+    font-size: 16px;
+}
+
+/* Success/Warning messages in tabs */
+.stTabs [data-baseweb="tab-panel"] .stSuccess,
+.stTabs [data-baseweb="tab-panel"] .stWarning {
+    margin-top: 1rem;
+    margin-bottom: 1rem;
+    border-radius: 8px;
+}
+
+/* Main Application Tabs */
+.tab-container {
+    background: linear-gradient(135deg, #1A202C, #2D3748);
+    border-radius: 16px;
+    padding: 20px;
+    margin: 20px 0;
+    box-shadow: 0 8px 16px rgba(0, 0, 0, 0.15);
+}
+
+.tab-content {
+    background: linear-gradient(135deg, #FFFFFF, #F8FAFC);
+    border-radius: 12px;
+    padding: 25px;
+    margin-top: 15px;
+    border: 1px solid rgba(226, 232, 240, 0.8);
+    box-shadow: 0 6px 12px rgba(0, 0, 0, 0.05);
+}
+
+/* Tab Content Typography */
+.tab-content h1, .tab-content h2, .tab-content h3 {
+    color: #2D3748;
+    margin-bottom: 1rem;
+    font-weight: 600;
+}
+
+.tab-content p {
+    color: #4A5568;
+    line-height: 1.8;
+    margin-bottom: 1rem;
+}
+
+/* Custom Scrollbar for Tab Content */
+.tab-content::-webkit-scrollbar {
+    width: 8px;
+}
+
+.tab-content::-webkit-scrollbar-track {
+    background: #F7FAFC;
+    border-radius: 4px;
+}
+
+.tab-content::-webkit-scrollbar-thumb {
+    background: linear-gradient(135deg, #CBD5E0, #A0AEC0);
+    border-radius: 4px;
+}
+
+.tab-content::-webkit-scrollbar-thumb:hover {
+    background: linear-gradient(135deg, #A0AEC0, #718096);
+}
+
+/* Enhanced Tab Indicators */
+.stTabs [data-baseweb="tab"][aria-selected="true"]::before {
+    content: '';
+    position: absolute;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100%;
+    background: rgba(255, 255, 255, 0.1);
+    border-radius: 8px;
+    z-index: -1;
+    animation: tabPulse 2s cubic-bezier(0.4, 0, 0.6, 1) infinite;
+}
+
+@keyframes tabPulse {
+    0%, 100% { opacity: 0.5; }
+    50% { opacity: 1; }
+}
+
+/* Text Inputs */
+.stTextInput > div {
+    background: #FFFFFF;
+}
+
+.stTextInput > div > div > input {
+    background: #F7FAFC;
+    border: 2px solid #E2E8F0;
+    border-radius: 10px;
+    padding: 12px 16px;
+    font-size: 15px;
+    color: #2D3748;
+    transition: all 0.3s ease;
+}
+
+.stTextInput > div > div > input:hover {
+    border-color: #CBD5E0;
+    background: #EDF2F7;
+}
+
+.stTextInput > div > div > input:focus {
+    border-color: #C53030;
+    box-shadow: 0 0 0 3px rgba(197, 48, 48, 0.2);
+    background: #FFFFFF;
+}
+
+/* Sidebar container styling - subtle modern gradient */
+[data-testid="stSidebar"] {
+    background: linear-gradient(135deg, #f8fafc, #f1f5f9);
+    color: #334155;
+    padding: 20px;
+    border-right: 1px solid rgba(148, 163, 184, 0.2);
+    box-shadow: 2px 0 8px rgba(0, 0, 0, 0.05);
+    transition: width 0.3s ease-in-out !important;
+}
+
+/* Collapsed sidebar styling */
+[data-testid="stSidebar"][aria-expanded="false"] {
+    margin-left: -21rem;
+}
+
+/* Sidebar title styling - improved contrast */
+[data-testid="stSidebar"] h1, [data-testid="stSidebar"] h2, [data-testid="stSidebar"] h3 {
+    color: #1e293b;
+    font-weight: 600;
+    margin-bottom: 1.5rem;
+    letter-spacing: 0.02em;
+    border-bottom: 2px solid #e2e8f0;
+    padding-bottom: 0.75rem;
+}
+
+/* Sidebar expander styling - modern and subtle */
+[data-testid="stSidebar"] .st-expander {
+    background: linear-gradient(135deg, #ffffff, #f8fafc);
+    border: 1px solid #e2e8f0;
+    border-radius: 8px;
+    margin-bottom: 1rem;
+    box-shadow: 0 2px 4px rgba(0, 0, 0, 0.03);
+    overflow: hidden;
+}
+
+[data-testid="stSidebar"] .st-expander > div:first-child {
+    color: #334155;
+    font-weight: 600;
+    padding: 0.875rem 1rem;
+    background: linear-gradient(135deg, #f8fafc, #f1f5f9);
+    border-bottom: 1px solid #e2e8f0;
+}
+
+/* Radio button styling - improved visibility */
+[data-testid="stSidebar"] .stRadio > div {
+    display: flex;
+    flex-direction: column;
+    gap: 0.625rem;
+}
+
+[data-testid="stSidebar"] .stRadio > div > label {
+    background: #ffffff;
+    color: #334155;
+    padding: 0.75rem 1rem;
+    border-radius: 6px;
+    font-weight: 500;
+    border: 1px solid #e2e8f0;
+    transition: all 0.2s ease;
+}
+
+[data-testid="stSidebar"] .stRadio > div > label:hover {
+    background: linear-gradient(135deg, #f1f5f9, #e2e8f0);
+    transform: translateY(-1px);
+    border-color: #cbd5e1;
+}
+
+[data-testid="stSidebar"] .stRadio > div > label[data-selected="true"] {
+    background: linear-gradient(135deg, #0ea5e9, #0284c7);
+    color: #ffffff;
+    border-color: #0284c7;
+    box-shadow: 0 2px 4px rgba(2, 132, 199, 0.2);
+}
+
+/* Input and select styling - improved contrast */
+[data-testid="stSidebar"] input, [data-testid="stSidebar"] select {
+    background: #ffffff;
+    color: #334155;
+    border: 1px solid #e2e8f0;
+    border-radius: 6px;
+    padding: 0.75rem;
+    font-size: 0.875rem;
+    margin-bottom: 0.75rem;
+    transition: all 0.2s ease;
+}
+
+[data-testid="stSidebar"] input:focus, [data-testid="stSidebar"] select:focus {
+    border-color: #0ea5e9;
+    box-shadow: 0 0 0 2px rgba(14, 165, 233, 0.1);
+    outline: none;
+}
+
+/* Button styling - modern and subtle */
+[data-testid="stSidebar"] button {
+    background: linear-gradient(135deg, #0ea5e9, #0284c7);
+    color: #ffffff;
+    border: none;
+    border-radius: 6px;
+    padding: 0.75rem 1rem;
+    font-weight: 500;
     cursor: pointer;
-    transition: color 0.3s ease;
+    transition: all 0.2s ease;
 }
 
-.sidebar-option:hover {
-    color: #1976D2;
+[data-testid="stSidebar"] button:hover {
+    background: linear-gradient(135deg, #0284c7, #0369a1);
+    transform: translateY(-1px);
+    box-shadow: 0 2px 4px rgba(2, 132, 199, 0.2);
+}
+
+/* Settings button styling */
+[data-testid="stSidebar"] .stButton > button {
+    background: linear-gradient(135deg, #3182CE, #2C5282);
+    color: white;
+    border: none;
+    padding: 0.75rem 1rem;
+    font-weight: 600;
+    border-radius: 8px;
+    transition: all 0.3s ease;
+    width: 100%;
+    margin-bottom: 1rem;
+}
+
+[data-testid="stSidebar"] .stButton > button:hover {
+    background: linear-gradient(135deg, #2C5282, #1A365D);
+    transform: translateY(-2px);
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+}
+
+/* Scrollbar styling - subtle and modern */
+[data-testid="stSidebar"]::-webkit-scrollbar {
+    width: 8px;
+}
+
+[data-testid="stSidebar"]::-webkit-scrollbar-track {
+    background: #f8fafc;
+}
+
+[data-testid="stSidebar"]::-webkit-scrollbar-thumb {
+    background: #cbd5e1;
+    border-radius: 4px;
+    border: 2px solid #f8fafc;
+}
+
+[data-testid="stSidebar"]::-webkit-scrollbar-thumb:hover {
+    background: #94a3b8;
 }
 
 /* Content section styling */
@@ -86,7 +392,6 @@ body {
     background-color: #ffffff;
 }
 
-
 /* Custom button styling */
 div.stButton > button:first-child {
     background: #1565C0;
@@ -203,3 +508,169 @@ select option {
     padding: 10px;
 }
 
+/* Content Planning Tools Styling */
+.content-header {
+    background: linear-gradient(135deg, #f8fafc, #f1f5f9);
+    padding: 1rem;
+    border-radius: 2px;
+    margin-bottom: 2rem;
+    border: 1px solid rgba(148, 163, 184, 0.2);
+    box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);
+}
+
+.content-header h2 {
+    color: #1e293b;
+    font-size: 1rem;
+    font-weight: 300;
+    margin-bottom: 0.5rem;
+}
+
+.content-header .subtitle {
+    color: #475569;
+    font-size: 1.1rem;
+    line-height: 1;
+}
+
+.tool-section {
+    background: white;
+    padding: 1rem;
+    border-radius: 2px;
+    margin-bottom: 1.5rem;
+    border: 1px solid #e2e8f0;
+}
+
+.tool-section h3 {
+    color: #1e293b;
+    font-size: 1.4rem;
+    font-weight: 600;
+    margin-bottom: 0.75rem;
+}
+
+.tool-section p {
+    color: #475569;
+    font-size: 1rem;
+    line-height: 1.5;
+}
+
+/* Button styling */
+.stButton > button {
+    background: linear-gradient(135deg, #3182ce, #2c5282);
+    color: white;
+    border: none;
+    padding: 0.75rem 1.5rem;
+    font-weight: 600;
+    border-radius: 8px;
+    transition: all 0.3s ease;
+}
+
+.stButton > button:hover {
+    background: linear-gradient(135deg, #2c5282, #1a365d);
+    transform: translateY(-2px);
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+}
+
+/* Search option containers styling */
+.search-option-container {
+    background: linear-gradient(135deg, #f8fafc, #f1f5f9);
+    border: 1px solid rgba(148, 163, 184, 0.2);
+    border-radius: 8px;
+    padding: 1rem;
+    margin-bottom: 1rem;
+    text-align: center;
+    height: 100%;
+    transition: all 0.3s ease;
+}
+
+.search-option-container:hover {
+    transform: translateY(-2px);
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.05);
+}
+
+.search-option-container h4 {
+    color: #1e293b;
+    margin-bottom: 0.5rem;
+}
+
+.search-option-container p {
+    color: #64748b;
+    margin: 0;
+}
+
+/* Button styling for search options */
+.stButton > button {
+    background: linear-gradient(135deg, #3182ce, #2c5282);
+    color: white;
+    border: none;
+    padding: 0.5rem 1rem;
+    font-weight: 600;
+    border-radius: 6px;
+    transition: all 0.3s ease;
+}
+
+.stButton > button:disabled {
+    background: linear-gradient(135deg, #94a3b8, #64748b);
+    cursor: not-allowed;
+}
+
+.stButton > button:not(:disabled):hover {
+    background: linear-gradient(135deg, #2c5282, #1e3a8a);
+    transform: translateY(-1px);
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+}
+
+/* Search options styling */
+.search-option {
+    background: linear-gradient(135deg, #f8fafc, #f1f5f9);
+    border: 1px solid rgba(148, 163, 184, 0.2);
+    border-radius: 8px;
+    padding: 1rem;
+    height: 100%;
+    transition: all 0.3s ease;
+    text-align: center;
+}
+
+.search-option:hover {
+    transform: translateY(-2px);
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.05);
+}
+
+.search-option h4 {
+    color: #1e293b;
+    margin-bottom: 0.5rem;
+    font-weight: 600;
+}
+
+.search-option p {
+    color: #64748b;
+    font-size: 0.9em;
+    margin-bottom: 1rem;
+}
+
+.search-option.active {
+    border: 2px solid #3182ce;
+    background: linear-gradient(135deg, #ebf8ff, #e6fffa);
+}
+
+/* Add these to your existing search-option styles */
+.search-option.disabled {
+    background: linear-gradient(135deg, #f1f5f9, #e2e8f0);
+    opacity: 0.8;
+    cursor: not-allowed;
+    border: 1px solid #cbd5e1;
+}
+
+
+.search-option .api-missing {
+    display: inline-block;
+    background: #fee2e2;
+    color: #dc2626;
+    padding: 2px 8px;
+    border-radius: 4px;
+    font-size: 0.8em;
+    margin-top: 0.5rem;
+}
+
+.search-option.disabled h4,
+.search-option.disabled p {
+    color: #64748b;
+}

pages/ai_research_setup_page.py

@@ -1,50 +0,0 @@
-"""Page for AI Research Setup redirection."""
-
-import streamlit as st
-from loguru import logger
-import sys
-import os
-
-# Configure logger
-logger.remove()  # Remove default handler
-logger.add(
-    "logs/ai_research_setup_page.log",
-    rotation="500 MB",
-    retention="10 days",
-    level="DEBUG",
-    format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {message}",
-    backtrace=True,
-    diagnose=True
-)
-logger.add(
-    sys.stdout,
-    level="INFO",
-    format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{message}</cyan>"
-)
-
-# Set page config
-st.set_page_config(
-    layout="wide",
-    initial_sidebar_state="collapsed",
-    menu_items={
-        'Get Help': None,
-        'Report a bug': None,
-        'About': None
-    }
-)
-
-def render_ai_research_setup_page():
-    """Render the AI Research Setup page."""
-    try:
-        logger.info("Starting AI Research Setup page")
-        
-        # Import and render the AI Research Setup component
-        from lib.utils.api_key_manager.components.ai_research_setup import render_ai_research_setup
-        render_ai_research_setup()
-        
-    except Exception as e:
-        logger.error(f"Error in render_ai_research_setup_page: {str(e)}")
-        st.error(f"An error occurred: {str(e)}")
-
-if __name__ == "__main__":
-    render_ai_research_setup_page() 

pages/personalization_setup.py

@@ -1,84 +0,0 @@
-import streamlit as st
-import os
-import json
-from pathlib import Path
-
-st.set_page_config(
-    page_title="Personalization Setup",
-    page_icon="⚙️",
-    layout="wide"
-)
-
-st.title("Personalization Setup")
-
-# Initialize session state for active tab if not exists
-if 'active_tab' not in st.session_state:
-    st.session_state.active_tab = "Writing Preferences"
-
-# Create tabs for different sections
-tab1, tab2 = st.tabs(["Writing Preferences", "AI Configuration"])
-
-with tab1:
-    st.write("""
-    This section allows you to customize your AI writing experience.
-    Configure your preferences and settings here.
-    """)
-
-    # Add your personalization options here
-    st.subheader("Writing Style Preferences")
-    tone = st.selectbox(
-        "Select your preferred writing tone",
-        ["Professional", "Casual", "Academic", "Creative"]
-    )
-
-    st.subheader("Content Preferences")
-    content_type = st.multiselect(
-        "Select your preferred content types",
-        ["Blog Posts", "Articles", "Social Media", "Technical Writing", "Creative Writing"]
-    )
-
-    if st.button("Save Preferences"):
-        st.success("Your preferences have been saved!")
-
-with tab2:
-    st.subheader("AI Configuration Settings")
-    
-    # Create a form for AI configuration
-    with st.form("ai_config_form"):
-        # API Keys
-        st.text_input("OpenAI API Key", type="password", key="openai_key")
-        st.text_input("Google API Key", type="password", key="google_key")
-        st.text_input("SerpAPI Key", type="password", key="serpapi_key")
-        
-        # Model Selection
-        st.selectbox("Select Model", ["gpt-3.5-turbo", "gpt-4"], key="model")
-        
-        # Temperature
-        st.slider("Temperature", 0.0, 2.0, 0.7, 0.1, key="temperature")
-        
-        # Max Tokens
-        st.number_input("Max Tokens", 100, 4000, 2000, 100, key="max_tokens")
-        
-        # Submit button
-        submitted = st.form_submit_button("Save Configuration")
-        
-        if submitted:
-            # Create config directory if it doesn't exist
-            config_dir = Path("config")
-            config_dir.mkdir(exist_ok=True)
-            
-            # Save configuration
-            config = {
-                "openai_key": st.session_state.openai_key,
-                "google_key": st.session_state.google_key,
-                "serpapi_key": st.session_state.serpapi_key,
-                "model": st.session_state.model,
-                "temperature": st.session_state.temperature,
-                "max_tokens": st.session_state.max_tokens
-            }
-            
-            config_file = config_dir / "test_config.json"
-            with open(config_file, "w") as f:
-                json.dump(config, f, indent=4)
-            
-            st.success("Configuration saved successfully!") 

requirements.txt

@@ -6,7 +6,7 @@ beautifulsoup4==4.12.2
 aiohttp>=3.11.11
 openai>=1.3.7
 PyPDF2>=3.0.1
-google-generativeai<0.9.0,>=0.8.0
+google-genai>=1.0.0
 anthropic>=0.18.1
 tenacity>=8.2.3
 tabulate>=0.9.0
@@ -31,7 +31,8 @@ prompt_toolkit>=3.0.43
 html2image>=2.0.5
 lxml[html_clean]>=5.3.0
 lxml_html_clean>=0.4.1
-streamlit>=1.29.0
+streamlit>=1.44.0
+Authlib>=1.3.2
 yfinance>=0.2.36
 pandas_ta>=0.3.14b0
 firecrawl-py>=1.14.1