app/services/fetcher.py

"""3-stage fetcher with full diagnostic output per stage."""

import logging
import re
import time
from typing import Any, Optional
from urllib.parse import urlparse

import httpx

logger = logging.getLogger(__name__)

USER_AGENT = (
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
)

HEADERS = {
    "User-Agent": USER_AGENT,
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9",
    "Accept-Encoding": "gzip, deflate, br, zstd",
    "Connection": "keep-alive",
    "Upgrade-Insecure-Requests": "1",
    "Sec-Fetch-Dest": "document",
    "Sec-Fetch-Mode": "navigate",
    "Sec-Fetch-Site": "none",
    "Sec-Fetch-User": "?1",
    "Sec-Ch-Ua": '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"',
    "Sec-Ch-Ua-Mobile": "?0",
    "Sec-Ch-Ua-Platform": '"Linux"',
    "Cache-Control": "max-age=0",
}

BLOCK_PATTERNS = {
    "cloudflare": r"(?i)(cloudflare|cf[-_]?ray|cf[-_]?browser|challenge[-_]?platform|turnstile|just[-_]?a[-_]?moment|checking[-_]?your[-_]?browser|enable[-_]?javascript|ddos[-_]?protection|attention[-_]?required|ray[-_]?id)",
    "captcha": r"(?i)(captcha|recaptcha|hcaptcha|arkose|funcaptcha|verify[-_]?you[-_]?are[-_]?human|prove[-_]?you[-_]?are[-_]?human|robot[-_]?check|bot[-_]?check|human[-_]?verification)",
    "rate_limit": r"(?i)(rate[-_]?limit|too[-_]?many[-_]?requests|429|throttl|slow[-_]?down|you[-_]?have[-_]?been[-_]?rate)",
    "access_denied": r"(?i)(access[-_]?denied|forbidden|blocked|not[-_]?allowed|unauthorized|ip[-_]?blocked|geo[-_]?blocked|blacklist|ban)",
    "waf": r"(?i)(waf|web[-_]?application[-_]?firewall|imperva|incapsula|sucuri|akamai[-_]?ghost|perimeterx|datadome|bot[-_]?management)",
    "login_required": r"(?i)(sign[-_]?in|log[-_]?in|login[-_]?required|authenticate|session[-_]?expired|please[-_]?log)",
    "js_challenge": r"(?i)(javascript[-_]?required|enable[-_]?javascript|browser[-_]?check|js[-_]?challenge|cookie[-_]?check)",
}


def _detect_blocks(body: str, headers: dict) -> list[dict]:
    hits = []
    header_str = str(headers).lower()
    for name, pattern in BLOCK_PATTERNS.items():
        body_match = re.search(pattern, body or "")
        header_match = re.search(pattern, header_str)
        if body_match or header_match:
            hits.append({
                "type": name,
                "in_body": body_match.group(0) if body_match else None,
                "in_headers": header_match.group(0) if header_match else None,
            })
    return hits


def _extract_title(html: str) -> str:
    m = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
    return m.group(1).strip() if m else ""


def _truncate(text: str, max_len: int = 3000) -> str:
    if len(text) <= max_len:
        return text
    return text[:max_len] + f"\n... [{len(text)} total chars]"


def _safe_headers(headers: dict) -> dict:
    safe = {}
    skip = {"set-cookie", "authorization", "x-api-key", "x-auth-token"}
    for k, v in headers.items():
        if k.lower() not in skip:
            safe[k] = v
    return safe


class FetchError(Exception):
    def __init__(self, message: str, stages: list[dict]):
        super().__init__(message)
        self.stages = stages


async def fetch_html(url: str, *, timeout: float = 30.0) -> tuple[str, str, str]:
    parsed = urlparse(url)
    if parsed.scheme not in ("http", "https"):
        url = "https://" + url

    stages: list[dict] = []

    # --- Stage 1: plain httpx ---
    t0 = time.time()
    try:
        async with httpx.AsyncClient(
            timeout=timeout,
            follow_redirects=True,
            headers=HEADERS,
        ) as client:
            resp = await client.get(url)
            elapsed_ms = int((time.time() - t0) * 1000)
            body = resp.text
            blocks = _detect_blocks(body, dict(resp.headers))
            stage = {
                "stage": "1-httpx",
                "method": "GET (httpx async)",
                "url": str(resp.url),
                "final_url": str(resp.url),
                "status": resp.status_code,
                "http_version": f"HTTP/{resp.http_version}",
                "elapsed_ms": elapsed_ms,
                "response_size_bytes": len(resp.content),
                "response_headers": _safe_headers(dict(resp.headers)),
                "title": _extract_title(body),
                "body_preview": _truncate(body),
                "redirects": [str(r.url) for r in resp.history] if resp.history else [],
                "blocks_detected": blocks if blocks else "none",
                "result": "success" if resp.status_code < 400 else "failed",
            }
            if resp.status_code < 400:
                stages.append(stage)
                return str(resp.url), body, _extract_title(body)
            stages.append(stage)
    except Exception as exc:
        elapsed_ms = int((time.time() - t0) * 1000)
        stages.append({
            "stage": "1-httpx",
            "method": "GET (httpx async)",
            "url": url,
            "status": None,
            "elapsed_ms": elapsed_ms,
            "error_type": type(exc).__name__,
            "error": str(exc),
            "result": "exception",
        })

    # --- Stage 2: playwright headless ---
    t0 = time.time()
    try:
        result = await _render_playwright(url, headless=True, timeout=timeout)
        elapsed_ms = int((time.time() - t0) * 1000)
        if result:
            final_url, html, title, diag = result
            diag["stage"] = "2-headless"
            diag["method"] = "Playwright headless Chromium"
            diag["elapsed_ms"] = elapsed_ms
            diag["result"] = "success"
            diag["blocks_detected"] = _detect_blocks(html, diag.get("response_headers", {}))
            stages.append(diag)
            return final_url, html, title
        stages.append({
            "stage": "2-headless",
            "method": "Playwright headless Chromium",
            "url": url,
            "elapsed_ms": elapsed_ms,
            "status": 403,
            "result": "failed",
            "note": "page returned HTTP 403 or errored",
        })
    except Exception as exc:
        elapsed_ms = int((time.time() - t0) * 1000)
        stages.append({
            "stage": "2-headless",
            "method": "Playwright headless Chromium",
            "url": url,
            "elapsed_ms": elapsed_ms,
            "error_type": type(exc).__name__,
            "error": str(exc),
            "result": "exception",
        })

    # --- Stage 3: playwright headed (xvfb) ---
    t0 = time.time()
    try:
        result = await _render_playwright(url, headless=False, timeout=timeout)
        elapsed_ms = int((time.time() - t0) * 1000)
        if result:
            final_url, html, title, diag = result
            diag["stage"] = "3-headed"
            diag["method"] = "Playwright headed Chromium (xvfb)"
            diag["elapsed_ms"] = elapsed_ms
            diag["result"] = "success"
            diag["blocks_detected"] = _detect_blocks(html, diag.get("response_headers", {}))
            stages.append(diag)
            return final_url, html, title
        stages.append({
            "stage": "3-headed",
            "method": "Playwright headed Chromium (xvfb)",
            "url": url,
            "elapsed_ms": elapsed_ms,
            "status": 403,
            "result": "failed",
            "note": "page returned HTTP 403 or errored",
        })
    except Exception as exc:
        elapsed_ms = int((time.time() - t0) * 1000)
        stages.append({
            "stage": "3-headed",
            "method": "Playwright headed Chromium (xvfb)",
            "url": url,
            "elapsed_ms": elapsed_ms,
            "error_type": type(exc).__name__,
            "error": str(exc),
            "result": "exception",
        })

    summary = "; ".join(
        f"[{s['stage']}] {s.get('status') or 'err'} {s.get('error','')}"
        for s in stages
    )
    raise FetchError(
        f"All fetch methods failed for {url}: {summary}",
        stages=stages,
    )


async def _render_playwright(
    url: str, headless: bool, timeout: float
) -> Optional[tuple[str, str, str, dict]]:
    try:
        from playwright.async_api import async_playwright
    except ImportError:
        return None

    async with async_playwright() as pw:
        browser = await pw.chromium.launch(headless=headless)
        try:
            ctx = await browser.new_context(
                user_agent=USER_AGENT,
                viewport={"width": 1280, "height": 800},
                locale="en-US",
            )

            console_logs = []
            network_errors = []
            all_requests = []

            # --- create page FIRST, then register events ---
            page = await ctx.new_page()

            page.on("console", lambda msg: console_logs.append({"type": msg.type, "text": msg.text}))
            page.on("requestfailed", lambda req: network_errors.append({"url": req.url, "failure": req.failure}))
            page.on("response", lambda resp: all_requests.append({
                "url": resp.url,
                "status": resp.status,
                "headers": _safe_headers(dict(resp.headers)),
            }))

            # --- STEP 1: initial navigation ---
            resp = await page.goto(
                url, wait_until="domcontentloaded", timeout=int(timeout * 1000)
            )

            status = resp.status if resp else None
            resp_headers = dict(resp.headers) if resp else {}
            initial_url = page.url
            initial_html = await page.content()
            initial_title = _extract_title(initial_html)
            initial_cookies = await ctx.cookies()

            # --- STEP 2: check if page is a challenge page ---
            is_challenge = bool(re.search(
                r"(?i)(challenge|captcha|turnstile|just.a.moment|checking.your.browser|verify.you.are.human|enable.javascript|cloudflare)",
                initial_html
            ))

            # --- STEP 3: wait for JS to execute ---
            post_js_html = None
            post_js_title = None
            if is_challenge:
                try:
                    await page.wait_for_timeout(8000)
                    post_js_html = await page.content()
                    post_js_title = _extract_title(post_js_html)
                except Exception:
                    pass

            # --- STEP 4: try clicking challenge button ---
            challenge_button_clicked = False
            try:
                btn = page.locator("input[type='submit'], button[type='submit'], #challenge-form button, .cf-turnstile button")
                if await btn.count() > 0:
                    await btn.first.click()
                    challenge_button_clicked = True
                    await page.wait_for_timeout(5000)
            except Exception:
                pass

            # --- STEP 5: final page state ---
            final_html = await page.content()
            final_title = _extract_title(final_html)
            final_url = page.url
            final_cookies = await ctx.cookies()

            # --- STEP 6: browser fingerprint ---
            fingerprint = {}
            try:
                fingerprint = await page.evaluate("""() => {
                    return {
                        webdriver: navigator.webdriver,
                        userAgent: navigator.userAgent,
                        appVersion: navigator.appVersion,
                        platform: navigator.platform,
                        languages: navigator.languages,
                        plugins: navigator.plugins.length,
                        hardwareConcurrency: navigator.hardwareConcurrency,
                        deviceMemory: navigator.deviceMemory,
                        maxTouchPoints: navigator.maxTouchPoints,
                        cookieEnabled: navigator.cookieEnabled,
                        doNotTrack: navigator.doNotTrack,
                        pdfViewerEnabled: navigator.pdfViewerEnabled,
                        chrome: typeof window.chrome !== 'undefined',
                        webdriver_flag: document.querySelector('[name="webdriver"]') !== null,
                        cdc_adoQpoasnfa76pfcZLmcfl_Array: typeof window.cdc_adoQpoasnfa76pfcZLmcfl_Array !== 'undefined',
                        cdc_adoQpoasnfa76pfcZLmcfl_Promise: typeof window.cdc_adoQpoasnfa76pfcZLmcfl_Promise !== 'undefined',
                        cdc_adoQpoasnfa76pfcZLmcfl_Symbol: typeof window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol !== 'undefined',
                        selenium_ide: typeof window._selenium_ide_recorder !== 'undefined',
                        phantom: typeof window._phantom !== 'undefined',
                        nightmare: typeof window.__nightmare !== 'undefined',
                        nightwatch: typeof window.__nightwatch !== 'undefined',
                    };
                }""")
            except Exception as e:
                fingerprint = {"error": str(e)}

            # --- STEP 7: DOM analysis ---
            dom_analysis = {}
            try:
                dom_analysis = await page.evaluate("""() => {
                    const iframes = Array.from(document.querySelectorAll('iframe')).map(f => ({
                        src: f.src, id: f.id, name: f.name, width: f.width, height: f.height,
                    }));
                    const scripts = Array.from(document.querySelectorAll('script[src]')).map(s => s.src);
                    const forms = Array.from(document.querySelectorAll('form')).map(f => ({
                        action: f.action, method: f.method, id: f.id,
                    }));
                    const inputs = Array.from(document.querySelectorAll('input')).map(i => ({
                        type: i.type, name: i.name, id: i.id, value: i.value ? i.value.substring(0, 100) : '',
                    }));
                    return {
                        iframes: iframes,
                        scripts: scripts,
                        forms: forms,
                        inputs: inputs,
                        bodyText: document.body ? document.body.innerText.substring(0, 2000) : '',
                        meta: Array.from(document.querySelectorAll('meta')).map(m => ({
                            name: m.getAttribute('name'),
                            httpEquiv: m.getAttribute('http-equiv'),
                            content: m.getAttribute('content'),
                        })),
                    };
                }""")
            except Exception as e:
                dom_analysis = {"error": str(e)}

            # --- STEP 8: screenshot ---
            screenshot_b64 = None
            try:
                screenshot_bytes = await page.screenshot(full_page=False)
                import base64
                screenshot_b64 = base64.b64encode(screenshot_bytes).decode("ascii")
            except Exception:
                pass

            diag = {
                "url": url,
                "initial_url": initial_url,
                "final_url": final_url,
                "status": status,
                "response_headers": _safe_headers(resp_headers),
                "response_size_bytes": len(initial_html.encode("utf-8")),
                "title_initial": initial_title,
                "title_final": final_title,
                "body_preview": _truncate(final_html),
                "initial_cookies": [{"name": c["name"], "domain": c["domain"], "httpOnly": c["httpOnly"], "secure": c["secure"], "sameSite": c["sameSite"]} for c in initial_cookies],
                "final_cookies": [{"name": c["name"], "domain": c["domain"], "httpOnly": c["httpOnly"], "secure": c["secure"], "sameSite": c["sameSite"]} for c in final_cookies],
                "new_cookies_after_challenge": [{"name": c["name"], "domain": c["domain"]} for c in final_cookies if c["name"] not in [ic["name"] for ic in initial_cookies]],
                "is_challenge_page": is_challenge,
                "challenge_button_clicked": challenge_button_clicked,
                "console_logs": console_logs[-30:] if console_logs else [],
                "network_errors": network_errors if network_errors else [],
                "all_requests_made": all_requests[:50] if all_requests else [],
                "fingerprint": fingerprint,
                "dom_analysis": dom_analysis,
                "screenshot_base64": screenshot_b64[:500] if screenshot_b64 else None,
            }

            if status and status >= 400:
                return None

            return final_url, final_html, final_title, diag
        finally:
            await browser.close()