Update Crawler: Add stealth args and challenge detection for Cloudflare

src/crawler.py

@@ -148,7 +148,7 @@ class Crawler:
 
     def _fetch_playwright(self, url):
         """
-        Fallback Strategy A: Use Playwright to render full page.
+        Fallback Strategy A: Use Playwright to render full page with Stealth Mode.
         """
         try:
             from playwright.sync_api import sync_playwright
@@ -160,54 +160,99 @@ class Crawler:
             # Acquire semaphore to prevent resource exhaustion
             with self.playwright_semaphore:
                 with sync_playwright() as p:
-                    # Launch options
-                    # headless=False can be useful for visual debugging if needed, but we keep True for automation.
-                    browser = p.chromium.launch(headless=True)
+                    # Stealth Args to bypass basic detection
+                    # We keep headless=True for performance, but mask it
+                    args = [
+                        "--disable-blink-features=AutomationControlled",
+                        "--no-sandbox",
+                        "--disable-setuid-sandbox",
+                        "--disable-dev-shm-usage",
+                        "--disable-accelerated-2d-canvas",
+                        "--no-first-run",
+                        "--no-zygote",
+                        "--disable-gpu",
+                        "--hide-scrollbars",
+                        "--mute-audio",
+                    ]
+                    
+                    browser = p.chromium.launch(headless=True, args=args)
                     
                     # Context with realistic viewport and locale
                     context = browser.new_context(
                         viewport={'width': 1920, 'height': 1080},
-                        user_agent=self.ua.random,
+                        user_agent=self.ua.random, # Rotate UA
                         locale="en-US",
                         timezone_id="America/New_York",
-                        java_script_enabled=True
+                        java_script_enabled=True,
+                        has_touch=True,
+                        is_mobile=False
                     )
+
+                    # --- KEY STEALTH INJECTION ---
+                    # Override navigator.webdriver to undefined
+                    context.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
+                    # mask chrome runtime
+                    context.add_init_script("window.chrome = { runtime: {} };")
+                    # -----------------------------
                     
                     page = context.new_page()
                     
                     # BLOCK RESOURCES for speed: images, fonts, css
-                    page.route("**/*.{png,jpg,jpeg,gif,svg,css,woff,woff2,ttf,otf}", lambda route: route.abort())
+                    # CAREFUL: Some WAFs check if CSS loads. If blocking persists, disable this.
+                    # For now, we block images/fonts but keep CSS/JS
+                    page.route("**/*.{png,jpg,jpeg,gif,svg,woff,woff2,ttf,otf}", lambda route: route.abort())
 
                     try:
                         logger.info(f"Playwright navigating to {url}...")
-                        page.goto(url, timeout=60000, wait_until="domcontentloaded")
+                        page.goto(url, timeout=90000, wait_until="domcontentloaded")
                         
-                        # Wait for a bit of hydration (Optimized for speed)
-                        page.wait_for_timeout(500)
+                        # Wait for hydration
+                        page.wait_for_timeout(2000)
                         
-                        # Log page title
+                        # --- Cloudflare / WAF Detection & Handling ---
                         title = page.title()
+                        content = page.content().lower()
+                        
+                        retries = 0
+                        while retries < 2 and ("just a moment..." in title.lower() or "challenge" in title.lower() or "verify you are human" in content):
+                            logger.warning(f"Detected Cloudflare Challenge ({title}). Waiting for 15s...")
+                            # Simulate mouse movement
+                            try:
+                                page.mouse.move(100, 100)
+                                page.mouse.down()
+                                page.mouse.up()
+                                page.mouse.move(200, 200)
+                            except:
+                                pass
+                                
+                            page.wait_for_timeout(15000)
+                            title = page.title()
+                            content = page.content().lower()
+                            retries += 1
+                        # ---------------------------------------------
+
                         logger.info(f"Page Title: {title}")
                         
-                        content = page.content()
+                        # Final content grab
+                        final_content = page.content()
                         
                         # Explicit cleanup
                         page.close()
                         context.close()
                         browser.close()
                         
-                        return content
+                        return final_content
                         
                     except Exception as nav:
                         logger.error(f"Playwright navigation error: {nav}")
                         return None
                     finally:
-                        # Ensure browser is closed even if errors occur
                         try:
                             browser.close()
                         except:
                             pass
 
+
         except Exception as e:
             logger.error(f"Playwright critical error: {e}")
             return None