Spaces:
Sleeping
Sleeping
File size: 2,142 Bytes
34d0dc9 d57e0f9 34d0dc9 d57e0f9 34d0dc9 d57e0f9 34d0dc9 d57e0f9 34d0dc9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | import random
from tenacity import (
retry,
stop_after_attempt,
wait_exponential,
retry_if_exception_type,
)
from curl_cffi.requests import AsyncSession
from loguru import logger
class BrowserClient:
def __init__(self):
self.session = AsyncSession()
self.count = 0
@retry(
stop=stop_after_attempt(3),
wait=wait_exponential(multiplier=2, min=2, max=10),
retry=retry_if_exception_type(Exception),
reraise=True,
)
async def request(self, url: str):
target_browsers = ["chrome110", "chrome120", "edge101", "chrome116"]
chosen_browser = random.choice(target_browsers)
response = await self.session.get(
url, impersonate=chosen_browser, timeout=30, allow_redirects=True
)
response.raise_for_status()
return response
async def fetch(self, url: str) -> str:
"""Fetch URL and return clean HTML. Raises exception on failure."""
task_log = logger.bind(context="browser_client", url=url)
try:
response = await self.request(url)
html = response.text
self.count += 1
if any(
term in html.lower() for term in ["cloudflare", "datadome", "captcha"]
):
task_log.warning(f"No {self.count} Bot detection triggered for {url}")
raise Exception("Bot Blocked")
return html
except Exception as e:
# Categorize common errors for the scraper_manager
error_str = str(e).lower()
if "404" in error_str or "not found" in error_str:
raise Exception("Wrong URL: 404 Not Found")
if "invalid url" in error_str or "no host" in error_str or "resolve host" in error_str:
raise Exception("Wrong URL: Invalid address")
task_log.error(f"No {self.count} Failed for {url}: {e}")
raise # Re-raise original exception if not categorized above
async def close(self):
await self.session.close()
|