crawler/client.py

import asyncio
import httpx
import logging
from typing import Optional, Dict, Any
from .config import CrawlerConfig
from .anti_blocking import AntiBlockingManager

logger = logging.getLogger("crawler.client")

class RobustHTTPClient:
    def __init__(self, config: CrawlerConfig, anti_blocking: AntiBlockingManager):
        self.config = config
        self.anti_blocking = anti_blocking
        
        # Configure client options
        limits = httpx.Limits(
            max_keepalive_connections=self.config.concurrency_limit,
            max_connections=self.config.concurrency_limit
        )
        
        # Select proxy if configured
        self.proxy_index = 0
        proxy = self._get_next_proxy()
        
        self.client = httpx.AsyncClient(
            proxy=proxy,
            limits=limits,
            timeout=httpx.Timeout(15.0, connect=5.0),
            follow_redirects=True,
            http2=True # Enable HTTP/2 for better stealth and multiplexing where supported
        )

    def _get_next_proxy(self) -> Optional[str]:
        if not self.config.proxies:
            return None
        proxy = self.config.proxies[self.proxy_index]
        self.proxy_index = (self.proxy_index + 1) % len(self.config.proxies)
        return proxy

    async def get(self, url: str, referer: Optional[str] = None) -> httpx.Response:
        """Performs a GET request with automatic headers, retries, and exponential backoff on HTTP 429."""
        headers = self.anti_blocking.generate_headers(self.config.domain, referer)
        
        retries = 3
        backoff_factor = 2.0
        
        for attempt in range(retries):
            try:
                # Add human-like random jitter delay before requesting (if not first run or to throttle)
                delay = self.anti_blocking.calculate_jitter_delay(self.config.delay_range)
                await asyncio.sleep(delay)
                
                logger.info(f"Requesting: {url} (Attempt {attempt + 1}/{retries})")
                
                # Perform the HTTP call
                response = await self.client.get(url, headers=headers)
                
                # Check for rate limiting
                if response.status_code == 429:
                    retry_after = response.headers.get("Retry-After")
                    sleep_time = float(retry_after) if retry_after and retry_after.isdigit() else (backoff_factor ** attempt) * 5.0
                    logger.warning(f"Rate limited (429) on {url}. Sleeping for {sleep_time}s before retry.")
                    await asyncio.sleep(sleep_time)
                    continue
                
                # Raise exception for 4xx/5xx errors
                response.raise_for_status()
                return response
                
            except httpx.HTTPStatusError as e:
                # HTTP error (e.g. 500, 503, 403, 404)
                status_code = e.response.status_code
                if status_code in [500, 502, 503, 504]:
                    # Server errors are worth retrying
                    sleep_time = (backoff_factor ** attempt) * 2.0
                    logger.warning(f"Server error {status_code} on {url}. Retrying in {sleep_time}s...")
                    await asyncio.sleep(sleep_time)
                    continue
                else:
                    # Client errors (403, 404, etc.) - don't retry, just propagate
                    logger.error(f"Client error {status_code} on {url}: {str(e)}")
                    raise e
                    
            except (httpx.RequestError, asyncio.TimeoutError) as e:
                # Connection timeouts, DNS failures, protocol errors
                sleep_time = (backoff_factor ** attempt) * 2.0
                logger.warning(f"Network error on {url}: {str(e)}. Retrying in {sleep_time}s...")
                await asyncio.sleep(sleep_time)
                continue
                
        raise httpx.RequestError(f"Failed to fetch {url} after {retries} attempts.")

    async def close(self):
        await self.client.aclose()