3-stage fetch: httpx, then headless Chromium, then headed Chromium (Xvfb); report exact per-stage results

app/services/fetcher.py

@@ -1,11 +1,14 @@
 """Safe async page fetching for the asset extractor.
 
-Guards: SSRF block-list, redirect cap, size cap, timeout, content-type check.
-
-Fallback: when a site answers 403 Forbidden or a Cloudflare-style challenge
-page, retry once with a headless-Chromium render. This distinguishes a
-client-fingerprint block (render succeeds) from a network/IP block
-(render also fails).
+Stage cascade:
+  1. httpx GET with browser-like headers (fast path).
+  2. Headless Chromium render (Playwright) — bypasses header/fingerprint blocks.
+  3. Headed Chromium render (Playwright under Xvfb, real browser window) —
+     bypasses headless detection.
+
+Every stage is attempted in order and reports its exact result. If all stages
+fail, the FetchError message contains the per-stage result so the 502 response
+(and container logs) show precisely where each method was blocked.
 """
 
 import asyncio
@@ -31,6 +34,21 @@ class FetchError(Exception):
     """Raised when the page cannot be safely fetched."""
 
 
+class StageResult:
+    """Exact outcome of one fetch attempt, for diagnostics."""
+
+    def __init__(self, stage: str, ok: bool, detail: str, bytes_=None):
+        self.stage = stage
+        self.ok = ok
+        self.detail = detail
+        self.bytes = bytes_
+
+    def __str__(self) -> str:
+        status = "OK" if self.ok else "FAIL"
+        suffix = f" ({self.bytes}b)" if self.bytes is not None else ""
+        return f"{self.stage}: {status}{suffix} - {self.detail}"
+
+
 def _blocked(raw_url: str) -> str | None:
     """Return a reason string if the URL host is disallowed, else None."""
     parsed = urlparse(raw_url)
@@ -71,37 +89,54 @@ def _looks_like_challenge(html: str) -> bool:
     return any(m in low for m in markers)
 
 
-async def _render_with_chromium(url: str) -> tuple[str, str]:
-    """Load the URL in headless Chromium and return (final_url, html).
+def _launch_args(headless: bool) -> list[str]:
+    args = ["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"]
+    if not headless:
+        # Reduce automation fingerprints in the headed browser.
+        args += [
+            "--disable-blink-features=AutomationControlled",
+            "--window-size=1366,768",
+        ]
+    return args
+
+
+async def _render_with_chromium(url: str, headless: bool, stage: str) -> tuple[str, str]:
+    """Load the URL in Chromium and return (final_url, html).
 
-    Runs in a thread so the (blocking) sync Playwright API doesn't stall
-    the event loop. Raises FetchError if the render itself is refused.
+    Runs in a thread. Raises FetchError (with per-stage detail) on refusal.
     """
-    logger.info("render fallback starting url=%r", url)
 
     def _render() -> tuple[str, str]:
         from playwright.sync_api import sync_playwright
 
         with sync_playwright() as p:
             browser = p.chromium.launch(
-                headless=True,
-                args=["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"],
+                headless=headless,
+                args=_launch_args(headless),
             )
             try:
                 context = browser.new_context(
                     user_agent=ASSETS_USER_AGENT,
                     locale="en-US",
                 )
+                # Mask the automation flag that anti-bot JS looks for.
+                context.add_init_script(
+                    "Object.defineProperty(navigator, 'webdriver', {get: () => undefined});"
+                )
                 page = context.new_page()
-                resp = page.goto(url, wait_until="domcontentloaded", timeout=int(ASSETS_RENDER_TIMEOUT * 1000))
+                resp = page.goto(
+                    url,
+                    wait_until="domcontentloaded",
+                    timeout=int(ASSETS_RENDER_TIMEOUT * 1000),
+                )
                 if resp is not None and resp.status == 403:
-                    raise FetchError(f"render returned HTTP 403")
+                    raise FetchError(f"{stage}: HTTP 403")
                 final_url = page.url
-                page.wait_for_timeout(1500)  # let JS-text pages settle
+                page.wait_for_timeout(2500)  # let JS/bot pages settle
                 html = page.content()
                 if len(html) > ASSETS_MAX_BYTES:
                     html = html[:ASSETS_MAX_BYTES]
-                logger.info("render fallback ok url=%r bytes=%d", final_url, len(html))
+                logger.info("%s ok url=%r final=%r bytes=%d", stage, url, final_url, len(html))
                 return final_url, html
             finally:
                 browser.close()
@@ -111,73 +146,118 @@ async def _render_with_chromium(url: str) -> tuple[str, str]:
     except FetchError:
         raise
     except Exception as exc:
-        raise FetchError(f"render failed: {exc}") from exc
+        raise FetchError(f"{stage}: {exc}") from exc
+
+
+async def _stage1_httpx(url: str) -> tuple[str, str]:
+    """Fast path: plain HTTP GET with browser headers. Returns (final_url, html)."""
+    headers = _browser_headers()
+    limits = httpx.Limits(max_connections=50, max_keepalive_connections=10)
+
+    async with httpx.AsyncClient(
+        follow_redirects=True,
+        max_redirects=ASSETS_MAX_REDIRECTS,
+        timeout=ASSETS_FETCH_TIMEOUT,
+        limits=limits,
+        headers=headers,
+    ) as client:
+        async with client.stream("GET", url) as resp:
+            resp.raise_for_status()
+
+            content_type = resp.headers.get("content-type", "").lower()
+            if "html" not in content_type and "xml" not in content_type:
+                raise FetchError(f"not an HTML page (content-type: {content_type or 'unknown'})")
+
+            chunks: list[bytes] = []
+            size = 0
+            async for chunk in resp.aiter_bytes():
+                size += len(chunk)
+                if size > ASSETS_MAX_BYTES:
+                    raise FetchError(f"page exceeds {ASSETS_MAX_BYTES} bytes")
+                chunks.append(chunk)
+
+            return str(resp.url), b"".join(chunks).decode("utf-8", "ignore")
 
 
 async def fetch_html(url: str) -> tuple[str, str, str]:
     """Fetch a page and return (final_url, html, title).
 
-    Raises FetchError on any unsafe/failed/oversized/non-HTML fetch.
+    Tries stage 1 (httpx), then stage 2 (headless Chromium), then stage 3
+    (headed Chromium under Xvfb). Reports the exact result of every stage in
+    the raised FetchError.
     """
 
-    # Basic SSRF / scheme guard on the requested URL.
     reason = _blocked(url)
     if reason:
         raise FetchError(reason)
 
-    headers = _browser_headers()
-
-    limits = httpx.Limits(max_connections=50, max_keepalive_connections=10)
+    stages: list[StageResult] = []
 
+    # ---- Stage 1: plain HTTP ----
     try:
-        async with httpx.AsyncClient(
-            follow_redirects=True,
-            max_redirects=ASSETS_MAX_REDIRECTS,
-            timeout=ASSETS_FETCH_TIMEOUT,
-            limits=limits,
-            headers=headers,
-        ) as client:
-            async with client.stream("GET", url) as resp:
-                resp.raise_for_status()
-
-                content_type = resp.headers.get("content-type", "").lower()
-                if "html" not in content_type and "xml" not in content_type:
-                    raise FetchError(f"not an HTML page (content-type: {content_type or 'unknown'})")
-
-                # Enforce size cap while streaming.
-                chunks: list[bytes] = []
-                size = 0
-                async for chunk in resp.aiter_bytes():
-                    size += len(chunk)
-                    if size > ASSETS_MAX_BYTES:
-                        raise FetchError(f"page exceeds {ASSETS_MAX_BYTES} bytes")
-                    chunks.append(chunk)
-
-                final_url = str(resp.url)
-                html = b"".join(chunks).decode("utf-8", "ignore")
-    except httpx.HTTPStatusError as exc:
-        if exc.response.status_code == 403 and ASSETS_RENDER_FALLBACK:
-            logger.info("fetch got 403 for url=%r -> trying render fallback", url)
-            final_url, html = await _render_with_chromium(url)
+        final_url, html = await _stage1_httpx(url)
+        if _looks_like_challenge(html):
+            stages.append(StageResult("1-httpx", True, "challenge page returned", len(html)))
+            fallthrough = True
         else:
-            raise FetchError(f"HTTP error {exc.response.status_code}") from exc
+            stages.append(StageResult("1-httpx", True, "html fetched", len(html)))
+            _guard_final(final_url)
+            return final_url, html, _extract_title(html)
+    except httpx.HTTPStatusError as exc:
+        sc = exc.response.status_code
+        stages.append(StageResult("1-httpx", False, f"HTTP {sc}"))
+        if not (ASSETS_RENDER_FALLBACK and sc in (403, 429, 503)):
+            raise FetchError(_summarize(stages)) from exc
+        fallthrough = True
+    except FetchError as exc:
+        stages.append(StageResult("1-httpx", False, str(exc)))
+        if str(exc).startswith("HTTP ") or "not an HTML" in str(exc):
+            raise FetchError(_summarize(stages)) from exc
+        raise FetchError("; ".join([str(s) for s in stages]) + " | stage1 failed") from exc
     except httpx.HTTPError as exc:
-        raise FetchError(f"request failed: {exc}") from exc
-
-    # The request may have "succeeded" (200) but returned a challenge page.
-    if _looks_like_challenge(html) and ASSETS_RENDER_FALLBACK:
-        logger.info("fetch got challenge page for url=%r -> trying render fallback", url)
-        rendered_url, rendered_html = await _render_with_chromium(url)
-        if not _looks_like_challenge(rendered_html):
-            final_url, html = rendered_url, rendered_html
-
-    # Guard against redirected-to-private hosts (defence in depth).
+        stages.append(StageResult("1-httpx", False, f"request failed: {exc}"))
+        if not ASSETS_RENDER_FALLBACK:
+            raise FetchError(_summarize(stages)) from exc
+        fallthrough = True
+
+    # ---- Stage 2: headless Chromium ----
+    if fallthrough:
+        try:
+            final_url, html = await _render_with_chromium(url, headless=True, stage="2-headless")
+            if _looks_like_challenge(html):
+                stages.append(StageResult("2-headless", True, "challenge page returned", len(html)))
+            else:
+                stages.append(StageResult("2-headless", True, "rendered OK", len(html)))
+                _guard_final(final_url)
+                return final_url, html, _extract_title(html)
+        except FetchError as exc:
+            stages.append(StageResult("2-headless", False, str(exc)))
+
+        # ---- Stage 3: headed Chromium under Xvfb ----
+        try:
+            final_url, html = await _render_with_chromium(url, headless=False, stage="3-headed")
+            if _looks_like_challenge(html):
+                stages.append(StageResult("3-headed", True, "challenge page returned", len(html)))
+            else:
+                stages.append(StageResult("3-headed", True, "rendered OK", len(html)))
+                _guard_final(final_url)
+                return final_url, html, _extract_title(html)
+        except FetchError as exc:
+            stages.append(StageResult("3-headed", False, str(exc)))
+
+        raise FetchError(_summarize(stages))
+
+    raise FetchError(_summarize(stages))
+
+
+def _guard_final(final_url: str) -> None:
     reason = _blocked(final_url)
     if reason:
         raise FetchError(f"redirected to {reason}")
 
-    title = _extract_title(html)
-    return final_url, html, title
+
+def _summarize(stages: list[StageResult]) -> str:
+    return " | ".join(str(s) for s in stages)
 
 
 def _extract_title(html: str) -> str | None: