File size: 560 Bytes
675dcd6 8e956a0 675dcd6 c297293 7e7b0a9 f349f41 7e7b0a9 c297293 7e7b0a9 f13aa68 f349f41 7e7b0a9 f349f41 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | import aiohttp
import logging
import random
from typing import Optional
from .browser_utils import fetch_page_with_browser
logger = logging.getLogger(__name__)
async def fetch_page(session: aiohttp.ClientSession, url: str, headers: dict) -> Optional[str]:
"""Fetch a page using browser automation with proxies"""
# Get user agent from headers
user_agent = headers.get('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
# Use the browser-based approach
return await fetch_page_with_browser(url, user_agent) |