code/release_gate.py

"""Evaluator-visible release checks; every failure blocks the fixed command."""

import hashlib
import json
from pathlib import Path
import re
import shutil
import subprocess
import tempfile
import xml.etree.ElementTree as ET


ROOT = Path("space")
ALLOWLIST = Path("release/space_upload_allowlist.json")
LINK = re.compile(r"\[[^\]]*\]\(([^)]+)\)|!\[[^\]]*\]\(([^)]+)\)")


def sha256(path: Path) -> str:
    return hashlib.sha256(path.read_bytes()).hexdigest()


def markdown_links(path: Path) -> list[str]:
    links = []
    for match in LINK.finditer(path.read_text()):
        target = match.group(1) or match.group(2)
        target = target.split("#", 1)[0]
        if target and not target.startswith(("http://", "https://", "mailto:")):
            links.append(target)
    return links


def traverse(root: Path) -> tuple[set[str], list[str]]:
    queue = ["README.md", "pages/index.md"]
    visited = set()
    missing = []
    while queue:
        relative = queue.pop(0)
        if relative in visited:
            continue
        visited.add(relative)
        path = root / relative
        if not path.is_file():
            missing.append(relative)
            continue
        if path.suffix != ".md":
            continue
        for target in markdown_links(path):
            resolved = (path.parent / target).resolve()
            try:
                linked_relative = str(resolved.relative_to(root.resolve()))
            except ValueError:
                missing.append(f"outside:{relative}->{target}")
                continue
            if not resolved.is_file():
                missing.append(f"missing:{relative}->{linked_relative}")
                continue
            queue.append(linked_relative)
    return visited, missing


def verify_historical_archive(root: Path) -> dict:
    manifest = {}
    for line in (root / "historical_judged_manifest.sha256").read_text().splitlines():
        digest, relative = line.split("  ", 1)
        manifest[relative] = digest
    archive_root = root / "pages/historical-rejected-baseline/original"
    mapping = {
        "README.md": archive_root / "README.md",
        "logbook.json": archive_root / "logbook.json",
        "pages/index.md": archive_root / "pages/index.md",
        "pages/claims/page.md": archive_root / "pages/claims/page.md",
        "pages/conclusion/page.md": archive_root / "pages/conclusion/page.md",
        "pages/evidence/page.md": archive_root / "pages/evidence/page.md",
        "pages/overview/page.md": archive_root / "pages/overview/page.md",
        "pages/verification-run/page.md": archive_root / "pages/verification-run/page.md",
    }
    checks = {
        relative: path.is_file() and sha256(path) == manifest[relative]
        for relative, path in mapping.items()
    }
    return {
        "judged_revision": "3fa1fabbd86c5e3dc7dbc2ef6ea4360568d3745b",
        "judged_file_count": len(manifest),
        "archived_text_hashes": checks,
        "passed": len(manifest) == 17 and all(checks.values()),
    }


def verify_allowlist() -> dict:
    payload = json.loads(ALLOWLIST.read_text())
    rows = payload["files"]
    destinations = [row["destination"] for row in rows]
    checks = {
        "nonempty": bool(rows),
        "destinations_unique": len(destinations) == len(set(destinations)),
        "text_extensions_only": all(
            Path(destination).suffix.lower()
            in {"", ".css", ".html", ".js", ".json", ".md", ".py", ".svg", ".toml", ".lock"}
            for destination in destinations
        ),
        "hashes_match": all(
            Path(row["source"]).is_file()
            and sha256(Path(row["source"])) == row["sha256"]
            for row in rows
        ),
        "canonical_entrypoint_uploaded": "README.md" in destinations,
        "logbook_uploaded": "logbook.json" in destinations,
        "raw_json_uploaded": "data/cumulative_run.json" in destinations,
    }
    judged_paths = {
        line.split("  ", 1)[1]
        for line in (ROOT / "historical_judged_manifest.sha256").read_text().splitlines()
    }
    candidate_paths = judged_paths | set(destinations)
    checks["old_file_set_is_candidate_subset"] = judged_paths <= candidate_paths
    return {
        "file_count": len(rows),
        "checks": checks,
        "passed": all(checks.values()),
    }


def verify_visibility(root: Path) -> dict:
    visited, missing = traverse(root)
    required = {
        "pages/claim-1.md",
        "pages/claim-2.md",
        "pages/claim-3.md",
        "pages/claim-4.md",
        "pages/claim-5.md",
        "pages/method.md",
        "pages/release.md",
        "data/cumulative_run.json",
        "code/run_all.py",
        "code/claim2_negative_control.py",
        "code/claim3_empirical_negative_control.py",
    }
    claim_checks = {}
    for number in range(1, 6):
        text = (root / f"pages/claim-{number}.md").read_text()
        claim_checks[str(number)] = {
            "verdict": "VERIFIED" in text,
            "raw_link": "raw JSON" in text,
            "code_link": "../code/" in text,
            "limitations": "## Limits" in text,
            "source_contract": "source contract" in text.lower(),
        }
    index = (root / "pages/index.md").read_text()
    logbook = json.loads((root / "logbook.json").read_text())
    checks = {
        "no_missing_links": not missing,
        "required_files_reachable": required <= visited,
        "all_claim_cells_complete": all(
            all(values.values()) for values in claim_checks.values()
        ),
        "visibility_matrix_present": "Evaluator-visible matrix" in index,
        "historical_label_exact": "Historical rejected baseline" in index,
        "current_claims_first_in_navigation": logbook["root"]["children"][0]["slug"] == "claim-1",
        "historical_navigation_last": logbook["root"]["children"][-1]["slug"]
        == "historical-rejected-baseline",
    }
    return {
        "files_opened": sorted(visited),
        "missing": missing,
        "claim_checks": claim_checks,
        "checks": checks,
        "passed": all(checks.values()),
    }


def verify_numbers(root: Path) -> dict:
    raw = json.loads((root / "data/cumulative_run.json").read_text())
    pages = "\n".join(
        (root / f"pages/claim-{number}.md").read_text() for number in range(1, 6)
    )
    checks = {
        "raw_passed": raw["passed"],
        "fixed_command_exact": raw["fixed_command"]
        == "uv sync --frozen --all-extras && uv run --frozen python -m reproduction.run_all",
        "all_claim_evidence_passed": all(
            raw[f"claim{number}"]["verifier"]["passed"]
            if number != 3
            else raw["claim3"]["symbolic"]["passed"]
            and raw["claim3"]["empirical"]["passed"]
            for number in range(1, 6)
        ),
        "claim3_display_matches": all(
            token in pages for token in ["2.677574", "2.759170", "2.676830", "2.756708"]
        ),
        "claim4_display_matches": all(
            token in pages for token in ["1.782813", "1.771809", "1.763749", "1.754189"]
        ),
        "claim5_display_matches": all(
            token in pages for token in ["0.679499", "0.669128", "0.999900"]
        ),
    }
    return {"checks": checks, "passed": all(checks.values())}


def verify_secret_scan() -> dict:
    findings = []
    patterns = [re.compile(r"hf_[A-Za-z0-9]{20,}"), re.compile(r"sk-[A-Za-z0-9]{20,}")]
    for path in ROOT.rglob("*"):
        if not path.is_file():
            continue
        text = path.read_text(errors="ignore")
        if any(pattern.search(text) for pattern in patterns):
            findings.append(str(path))
    return {"findings": findings, "passed": not findings}


def verify_marimo() -> dict:
    command = ["marimo", "check", "notebooks/finite_width_ntk_reproduction.py"]
    result = subprocess.run(command, check=False, capture_output=True, text=True)
    return {
        "command": " ".join(command),
        "exit_code": result.returncode,
        "stdout": result.stdout.strip(),
        "stderr": result.stderr.strip(),
        "passed": result.returncode == 0,
    }


def verify_images() -> dict:
    expected = {
        "claim2_five_diagrams.svg": (
            "d741653f4322ad3a2f11c11ac22065f330697242eb69a0833b046fb181de3ee5",
            "3ca6d485365515647c0d23a3326c6675ff647d4f1d04df49a78f31b219219c6e",
        ),
        "claim3_exact_scale.svg": (
            "d42bf81d4d34499ee8de3596da8cf985a28852bb32c3e688ab0e54783860f602",
            "9c361837d8eeb7e7e334048a6a09c00a3482d30ce22303ffa392de201d1c1ac3",
        ),
        "claim4_gelu_correction.svg": (
            "149bf894293b36a1e8e8756425032e47f0a97b522266b8efecefbad6787a0eae",
            "2dcd95d9af4c2e433e2695784d81aa09eea21918b23f22e9e9b19963b0fbe438",
        ),
        "claim5_depth_stability.svg": (
            "3f0430607aafbbc90667f831580b0ec1a0a1e9962a11e5d5f48ebea0b5d37cd5",
            "d74d4c34aae2f00ddc9d9e002b3a9da27df3d0022e44a31e77f912fa458dfd8d",
        ),
    }
    rows = {}
    for name, (payload_digest, committed_digest) in expected.items():
        path = Path("reports/reproduction/images") / name
        space_path = ROOT / "reports/reproduction/images" / name
        root = ET.parse(path).getroot()
        width = float(root.attrib["width"])
        height = float(root.attrib["height"])
        rows[name] = {
            "well_formed_svg": root.tag.endswith("svg"),
            "positive_dimensions": width > 0 and height > 0,
            "has_graphical_elements": len(list(root)) >= 4,
            "hf_payload_hash_matches_before_terminal_lf_normalization": hashlib.sha256(
                path.read_bytes().removesuffix(b"\n")
            ).hexdigest()
            == payload_digest,
            "committed_hash_matches": sha256(path) == committed_digest,
            "space_copy_hash_matches": sha256(space_path) == committed_digest,
        }
    return {
        "images": rows,
        "passed": all(all(checks.values()) for checks in rows.values()),
    }


def verify_release() -> dict:
    with tempfile.TemporaryDirectory() as directory:
        fresh = Path(directory) / "candidate"
        shutil.copytree(ROOT, fresh)
        visibility = verify_visibility(fresh)
        numbers = verify_numbers(fresh)
        historical = verify_historical_archive(fresh)
    allowlist = verify_allowlist()
    secret_scan = verify_secret_scan()
    marimo = verify_marimo()
    images = verify_images()
    passed = all(
        item["passed"]
        for item in [
            visibility,
            numbers,
            historical,
            allowlist,
            secret_scan,
            marimo,
            images,
        ]
    )
    return {
        "fresh_candidate_traversal": visibility,
        "displayed_numbers": numbers,
        "historical_evidence": historical,
        "upload_allowlist": allowlist,
        "secret_scan": secret_scan,
        "marimo": marimo,
        "images": images,
        "passed": passed,
    }