New: AI SEO tools, OpenGraph Tags generator

2024-08-09 16:17:09 +05:30
parent 4d887c87eb
commit 20bda6c964
5 changed files with 133 additions and 10 deletions
--- a/lib/ai_web_researcher/firecrawl_web_crawler.py
+++ b/lib/ai_web_researcher/firecrawl_web_crawler.py
@@ -61,13 +61,14 @@ def scrape_url(url):
        dict: The result of the URL scraping, or None if an error occurred.
    """
    client = initialize_client()
-    params = {
-    'pageOptions': {
-        'onlyMainContent': True
-        }
-    }
+    #params = {
+    #'pageOptions': {
+    #    'onlyMainContent': True
+    #    }
+    #}
    try:
-        result = client.scrape_url(url, params=params)
+        #result = client.scrape_url(url, params=params)
+        result = client.scrape_url(url)
        return result
    except Exception as e:
        logging.error(f"Error scraping URL: {e}")