Recovered state: integrated TrendSurferAgent, restored frontend/backend files, and cleaned up recovery scripts
This commit is contained in:
@@ -23,7 +23,7 @@ class WebCrawlerLogic:
|
||||
self.headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||
}
|
||||
self.timeout = 30
|
||||
self.timeout = 45 # Increased from 30 to 45 seconds for slower sites
|
||||
self.max_content_length = 10000
|
||||
|
||||
def _validate_url(self, url: str) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user