Spaces:
Running
Running
AI Bot commited on
Commit ·
a7bac51
1
Parent(s): 9d40235
Increase HTTP timeouts to prevent fallback to slow Playwright
Browse files- src/crawler.py +3 -3
src/crawler.py
CHANGED
|
@@ -62,7 +62,7 @@ class Crawler:
|
|
| 62 |
# Tier 2: Cloudscraper (Intermediate, handles JS challenges)
|
| 63 |
logger.info(f"Tier 1 failed. Trying Cloudscraper info for {url}...")
|
| 64 |
try:
|
| 65 |
-
resp = self.scraper.get(url, timeout=
|
| 66 |
if 200 <= resp.status_code < 300:
|
| 67 |
if not self._is_blocked(resp.text):
|
| 68 |
return resp.text
|
|
@@ -119,9 +119,9 @@ class Crawler:
|
|
| 119 |
# Rotate User Agent
|
| 120 |
self.session.headers["User-Agent"] = self.ua.random
|
| 121 |
|
| 122 |
-
#
|
| 123 |
# verify=False is inherent in the session from init, but good to be explicit if needed (curl_cffi uses session setting)
|
| 124 |
-
response = self.session.get(url, timeout=
|
| 125 |
|
| 126 |
# Check for 200 OK (or close to it)
|
| 127 |
if 200 <= response.status_code < 300:
|
|
|
|
| 62 |
# Tier 2: Cloudscraper (Intermediate, handles JS challenges)
|
| 63 |
logger.info(f"Tier 1 failed. Trying Cloudscraper info for {url}...")
|
| 64 |
try:
|
| 65 |
+
resp = self.scraper.get(url, timeout=30)
|
| 66 |
if 200 <= resp.status_code < 300:
|
| 67 |
if not self._is_blocked(resp.text):
|
| 68 |
return resp.text
|
|
|
|
| 119 |
# Rotate User Agent
|
| 120 |
self.session.headers["User-Agent"] = self.ua.random
|
| 121 |
|
| 122 |
+
# Increased timeout to 30 seconds to handle slow sites without falling back to Playwright
|
| 123 |
# verify=False is inherent in the session from init, but good to be explicit if needed (curl_cffi uses session setting)
|
| 124 |
+
response = self.session.get(url, timeout=30, allow_redirects=True)
|
| 125 |
|
| 126 |
# Check for 200 OK (or close to it)
|
| 127 |
if 200 <= response.status_code < 300:
|