WIP - Streamlit UI, firecrawl - V0.5

2024-06-12 16:01:46 +05:30
parent ccbaa0e4fa
commit f2aa79264e
12 changed files with 201 additions and 261 deletions
--- a/lib/ai_web_researcher/firecrawl_web_crawler.py
+++ b/lib/ai_web_researcher/firecrawl_web_crawler.py
@@ -60,10 +60,14 @@ def scrape_url(url):
    Returns:
        dict: The result of the URL scraping, or None if an error occurred.
    """
-    print(f"\n\nURL: {url} ---- {os.getenv('FIRECRAWL_API_KEY')}\n\n")
    client = initialize_client()
+    params = {
+    'pageOptions': {
+        'onlyMainContent': True
+        }
+    }
    try:
-        result = client.scrape_url(url)
+        result = client.scrape_url(url, params=params)
        return result
    except Exception as e:
        logging.error(f"Error scraping URL: {e}")