scripts/eboard/try_scrape_with_cookies.py

#!/usr/bin/env python3
"""
Try loading a URL with Playwright after passing bot walls manually.

Use this when the meetings pipeline hits ``captcha:cloudflare_js_challenge``:
1. In Chrome, open the URL, complete Cloudflare / Turnstile if shown.
2. Export cookies (Cookie-Editor / EditThisCookie) as JSON array of
   {name, value, domain, path, ...} — save e.g. ``./cookies.json``.
3. Run this script with ``--cookies-json`` or use ``--storage-state`` from
   ``playwright codegen`` / ``await context.storage_state(path=...)``.

Examples (repo root, venv active)::

    .venv/bin/python scripts/development/try_scrape_with_cookies.py \\
        --url 'https://www.tuscaloosacityschools.com/about-us/calendar' \\
        --out /tmp/tcs_try.html

    .venv/bin/python scripts/development/try_scrape_with_cookies.py \\
        --url 'https://www.tuscaloosacityschools.com/about-us/board-of-education/meet-the-board-members' \\
        --cookies-json ./tusc_cookies.json \\
        --wait-ms 8000

    # After one successful manual session in Playwright:
    .venv/bin/python scripts/development/try_scrape_with_cookies.py \\
        --url 'https://www.tuscaloosacityschools.com/' \\
        --storage-state ./tusc_state.json

Env (optional, same spirit as meetings scraper)::

    SCRAPED_MEETINGS_PLAYWRIGHT_CHANNEL=chrome
    SCRAPED_MEETINGS_PLAYWRIGHT_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome
    TRY_SCRAPE_HEADLESS=false   # default is false (visible browser)
"""
from __future__ import annotations

import argparse
import asyncio
import json
import os
import sys
from pathlib import Path
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse


def _launch_kwargs() -> Dict[str, Any]:
    exe = (os.environ.get("SCRAPED_MEETINGS_PLAYWRIGHT_CHROMIUM_EXECUTABLE") or "").strip()
    ch = (os.environ.get("SCRAPED_MEETINGS_PLAYWRIGHT_CHANNEL") or "").strip().lower()
    kw: Dict[str, Any] = {"headless": _headless()}
    if exe and Path(exe).is_file():
        kw["executable_path"] = exe
    elif ch in ("chrome", "msedge", "chromium"):
        kw["channel"] = ch
    return kw


def _headless() -> bool:
    v = (os.environ.get("TRY_SCRAPE_HEADLESS") or "false").strip().lower()
    return v in ("1", "true", "yes", "on")


def _registrable_cookie_domain(host: str) -> str:
    """Best-effort eTLD+1 style domain with leading dot for Playwright (not a full PSL)."""
    h = (host or "").lower().strip()
    if not h:
        return ""
    parts = [p for p in h.split(".") if p]
    if len(parts) >= 2:
        return "." + ".".join(parts[-2:])
    return "." + h if not h.startswith(".") else h


def _normalize_playwright_cookies(raw: List[Dict[str, Any]], page_host: str) -> List[Dict[str, Any]]:
    """Map common browser-extension exports to Playwright's add_cookies shape."""
    out: List[Dict[str, Any]] = []
    fallback_domain = _registrable_cookie_domain(page_host)
    for c in raw:
        name = (c.get("name") or "").strip()
        value = c.get("value")
        if not name or value is None:
            continue
        raw_dom = (c.get("domain") or "").strip()
        if raw_dom.startswith("."):
            domain = raw_dom
        elif raw_dom:
            # Extension often stores host without leading dot
            domain = _registrable_cookie_domain(raw_dom.lstrip("."))
        else:
            domain = fallback_domain
        path = (c.get("path") or "/").strip() or "/"
        if not domain:
            domain = fallback_domain
        entry: Dict[str, Any] = {
            "name": name,
            "value": str(value),
            "domain": domain,
            "path": path,
        }
        if "secure" in c:
            entry["secure"] = bool(c["secure"])
        if c.get("httpOnly") is not None:
            entry["httpOnly"] = bool(c["httpOnly"])
        ss = c.get("sameSite")
        if ss in ("Strict", "Lax", "None"):
            entry["sameSite"] = ss
        exp = c.get("expirationDate")
        if isinstance(exp, (int, float)) and exp > 0:
            entry["expires"] = float(exp)
        out.append(entry)
    return out


async def _run(
    url: str,
    out_path: Path,
    cookies_json: Optional[Path],
    storage_state: Optional[Path],
    wait_ms: int,
) -> int:
    try:
        from playwright.async_api import async_playwright
    except ImportError:
        print("Install Playwright in this venv: pip install playwright && playwright install chromium", file=sys.stderr)
        return 1

    parsed = urlparse(url)
    host = (parsed.hostname or "").lower()

    cookies: Optional[List[Dict[str, Any]]] = None
    if cookies_json:
        data = json.loads(cookies_json.read_text(encoding="utf-8"))
        if isinstance(data, dict) and "cookies" in data:
            data = data["cookies"]
        if not isinstance(data, list):
            print("cookies JSON must be a list of cookie objects (or {\"cookies\": [...]})", file=sys.stderr)
            return 1
        cookies = _normalize_playwright_cookies(data, host)

    async with async_playwright() as p:
        browser = await p.chromium.launch(**_launch_kwargs())
        try:
            if storage_state and storage_state.is_file():
                ctx = await browser.new_context(
                    storage_state=str(storage_state),
                    viewport={"width": 1280, "height": 900},
                )
            else:
                ctx = await browser.new_context(viewport={"width": 1280, "height": 900})
            if cookies:
                await ctx.add_cookies(cookies)
            try:
                from playwright_stealth import Stealth

                page = await ctx.new_page()
                await Stealth().apply_stealth_async(page)
            except Exception:
                page = await ctx.new_page()

            resp = await page.goto(url, wait_until="domcontentloaded", timeout=120_000)
            await asyncio.sleep(max(0, wait_ms) / 1000.0)
            html = await page.content()
            title = await page.title()
            final = page.url
            st = resp.status if resp is not None else 0

            out_path.write_text(html, encoding="utf-8")
            print(f"status={st} title={title!r}")
            print(f"final_url={final}")
            print(f"wrote {len(html)} chars -> {out_path}")
            low = html[:12000].lower()
            if "cf-challenge" in low or "turnstile" in low or "just a moment" in low:
                print("note: page still looks like a bot wall — try visible Chrome (headless=false), real channel=chrome, or fresher cookies.", file=sys.stderr)
            return 0
        finally:
            await browser.close()


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--url", required=True, help="Page to open (after you export cookies from same origin if needed)")
    ap.add_argument("--out", type=Path, default=Path("/tmp/try_scrape_out.html"), help="Where to save raw HTML")
    ap.add_argument("--cookies-json", type=Path, default=None, help="JSON array of cookies from browser extension")
    ap.add_argument("--storage-state", type=Path, default=None, help="Playwright storage_state.json from a prior session")
    ap.add_argument("--wait-ms", type=int, default=4000, help="Extra wait after domcontentloaded (JS challenges)")
    args = ap.parse_args()
    if args.cookies_json and args.storage_state:
        print("Use only one of --cookies-json or --storage-state", file=sys.stderr)
        sys.exit(2)
    rc = asyncio.run(
        _run(
            args.url,
            args.out,
            args.cookies_json,
            args.storage_state,
            args.wait_ms,
        )
    )
    raise SystemExit(rc)


if __name__ == "__main__":
    main()