import asyncio import logging from typing import Optional from playwright.async_api import async_playwright, Browser, Playwright, BrowserContext import random import sys # Critical Fix for Windows: Force ProactorEventLoop for subprocess support if sys.platform == 'win32': asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) # Configure logging logger = logging.getLogger(__name__) # List of realistic User-Agents USER_AGENTS = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36" ] class BrowserManager: _instance = None def __new__(cls): if cls._instance is None: cls._instance = super(BrowserManager, cls).__new__(cls) cls._instance._initialized = False return cls._instance def __init__(self): if self._initialized: return self.playwright: Optional[Playwright] = None self.browser: Optional[Browser] = None # Limit concurrent scraping operations to prevent OOM # 3 concurrent tabs is a safe starting point for a typical container (1-2GB RAM) self.semaphore = asyncio.Semaphore(3) self._initialized = True logger.info("BrowserManager initialized (Semaphore: 3)") async def start(self): """Initialize the global browser instance""" if self.browser: return try: logger.info("Starting Playwright...") self.playwright = await async_playwright().start() self.browser = await self.playwright.chromium.launch( headless=True, # Add arguments to improve stability in container environments args=[ '--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', # Overcome limited /dev/shm size '--disable-gpu', # Not needed for headless '--single-process' # Required for memory compliance ] ) logger.info("Global Browser Instance Started successfully") except Exception as e: logger.error(f"Failed to start Playwright: {e}") raise async def shutdown(self): """Gracefully close the global browser instance""" logger.info("Shutting down BrowserManager...") if self.browser: await self.browser.close() self.browser = None if self.playwright: await self.playwright.stop() self.playwright = None logger.info("BrowserManager shutdown complete") async def get_content(self, url: str) -> Optional[str]: """ Fetch dynamic content using a fresh context from the shared browser. Controlled by semaphore to prevention resource exhaustion. """ if not self.browser: logger.error("Browser not initialized! Call start() first.") return None async with self.semaphore: context: Optional[BrowserContext] = None page = None try: # Create a lightweight context (incognito-like) # Randomize user agent for basic anti-bot evasion context = await self.browser.new_context( user_agent=random.choice(USER_AGENTS), viewport={'width': 1920, 'height': 1080}, java_script_enabled=True ) page = await context.new_page() logger.info(f"Navigating to {url}") # "domcontentloaded" is faster than "networkidle" and usually sufficient for text await page.goto(url, wait_until="domcontentloaded", timeout=15000) # Wait a bit for JS hydration (React/Next.js) await page.wait_for_timeout(2000) content = await page.content() logger.info(f"Successfully scraped {len(content)} bytes from {url}") return content except Exception as e: logger.error(f"Scraping failed for {url}: {e}") return None finally: # CRITICAL: Always close context to free memory if page: try: await page.close() except: pass if context: try: await context.close() except: pass # Global Singleton Instance browser_manager = BrowserManager()