AI Bot commited on
Commit
a7bac51
·
1 Parent(s): 9d40235

Increase HTTP timeouts to prevent fallback to slow Playwright

Browse files
Files changed (1) hide show
  1. src/crawler.py +3 -3
src/crawler.py CHANGED
@@ -62,7 +62,7 @@ class Crawler:
62
  # Tier 2: Cloudscraper (Intermediate, handles JS challenges)
63
  logger.info(f"Tier 1 failed. Trying Cloudscraper info for {url}...")
64
  try:
65
- resp = self.scraper.get(url, timeout=10)
66
  if 200 <= resp.status_code < 300:
67
  if not self._is_blocked(resp.text):
68
  return resp.text
@@ -119,9 +119,9 @@ class Crawler:
119
  # Rotate User Agent
120
  self.session.headers["User-Agent"] = self.ua.random
121
 
122
- # Reduced timeout to 5 seconds
123
  # verify=False is inherent in the session from init, but good to be explicit if needed (curl_cffi uses session setting)
124
- response = self.session.get(url, timeout=5, allow_redirects=True)
125
 
126
  # Check for 200 OK (or close to it)
127
  if 200 <= response.status_code < 300:
 
62
  # Tier 2: Cloudscraper (Intermediate, handles JS challenges)
63
  logger.info(f"Tier 1 failed. Trying Cloudscraper info for {url}...")
64
  try:
65
+ resp = self.scraper.get(url, timeout=30)
66
  if 200 <= resp.status_code < 300:
67
  if not self._is_blocked(resp.text):
68
  return resp.text
 
119
  # Rotate User Agent
120
  self.session.headers["User-Agent"] = self.ua.random
121
 
122
+ # Increased timeout to 30 seconds to handle slow sites without falling back to Playwright
123
  # verify=False is inherent in the session from init, but good to be explicit if needed (curl_cffi uses session setting)
124
+ response = self.session.get(url, timeout=30, allow_redirects=True)
125
 
126
  # Check for 200 OK (or close to it)
127
  if 200 <= response.status_code < 300: