engine/webcheck.py

"""Headless verification of model-built web apps (the web `run_python`).

smolbuilder's agent writes HTML/CSS/JS but, unlike the Python path, had no way
to *run* it — so it shipped broken apps and couldn't tell. This bridges to a
small Node + jsdom checker (engine/webcheck.js) that loads the page, runs its
scripts, clicks every button, and reports JavaScript errors.

Graceful degradation is deliberate: if Node or jsdom isn't available (e.g. a
minimal Space image), we return `None` ("unverifiable") rather than failing the
build — the agent/router fall back to the structural check.
"""
from __future__ import annotations

import json
import shutil
import subprocess
import tempfile
from pathlib import Path

_CHECKER = Path(__file__).with_name("webcheck.js")


def available() -> bool:
    """True if we can actually run the headless check (Node present)."""
    return shutil.which("node") is not None and _CHECKER.exists()


def check_html(html: str, timeout: int = 20) -> tuple[bool | None, list[str]]:
    """Run the headless check on an HTML document.

    Returns (ok, errors):
      - (True, [])        the app loaded and all buttons clicked without error
      - (False, [...])    real JavaScript errors were found
      - (None, [...])     unverifiable (Node/jsdom missing, or the checker broke)
    """
    node = shutil.which("node")
    if not node or not _CHECKER.exists():
        return None, ["node/jsdom unavailable (skipped runtime check)"]

    with tempfile.NamedTemporaryFile("w", suffix=".html", delete=False) as f:
        f.write(html)
        path = f.name
    try:
        proc = subprocess.run(
            [node, str(_CHECKER), path],
            capture_output=True, text=True, timeout=timeout,
        )
    except subprocess.TimeoutExpired:
        return None, [f"runtime check timed out after {timeout}s"]
    finally:
        Path(path).unlink(missing_ok=True)

    if proc.returncode == 3:        # jsdom not installed
        return None, ["jsdom not installed (skipped runtime check)"]
    line = (proc.stdout or "").strip().splitlines()
    if not line:
        return None, [f"runtime check produced no output: {proc.stderr.strip()[:200]}"]
    try:
        data = json.loads(line[-1])
    except json.JSONDecodeError:
        return None, [f"runtime check output unparseable: {line[-1][:200]}"]

    if data.get("ok") is None:
        return None, [data.get("infra", "unverifiable")]
    return bool(data.get("ok")), list(data.get("errors", []))