| import aiohttp | |
| import logging | |
| import random | |
| from typing import Optional | |
| from .browser_utils import fetch_page_with_browser | |
| logger = logging.getLogger(__name__) | |
| async def fetch_page(session: aiohttp.ClientSession, url: str, headers: dict) -> Optional[str]: | |
| """Fetch a page using browser automation with proxies""" | |
| # Get user agent from headers | |
| user_agent = headers.get('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36') | |
| # Use the browser-based approach | |
| return await fetch_page_with_browser(url, user_agent) |