bundle/v0/scripts/validate_trackio_logbook.py

from __future__ import annotations

import json
import re
import sys
from pathlib import Path


ROOT = Path(__file__).resolve().parents[1]
LOGBOOK = ROOT / ".trackio" / "logbook"
PAGES = LOGBOOK / "pages"
JOB_TOKEN = "{{HF_JOB_URL_PENDING_AFTER_APPROVED_LAUNCH}}"
BUNDLE_TOKEN = "{{REPRO_BUNDLE_ARTIFACT_CELL_INSERTED_BY_TRACKIO_PUBLISH}}"


def fail(message: str) -> None:
    raise AssertionError(message)


def main() -> int:
    model = json.loads((LOGBOOK / "logbook.json").read_text(encoding="utf-8"))
    metadata = json.loads((ROOT / ".trackio" / "metadata.json").read_text(encoding="utf-8"))
    children = model["root"]["children"]
    titles = [item["title"] for item in children]
    if titles[0] != "Executive summary" or titles[-1] != "Conclusion":
        fail("canonical first and last pages changed")
    if len(children) != 8 or sum(title.startswith("Claim ") for title in titles) != 6:
        fail("expected Executive summary, six claims, and Conclusion only")
    if metadata.get("tags") != ["icml2026-repro", "paper-K1EPPO9t2c"]:
        fail("challenge discovery tags changed")

    index = (PAGES / "index.md").read_text(encoding="utf-8")
    if "## Pages" not in index or "## Scope" in index or "Verdict" in index:
        fail("index contains findings or lacks the canonical Pages table")
    if "https://huggingface.co/papers/2509.12689" not in index:
        fail("paper link missing")

    all_text = index
    for item in children:
        page = (LOGBOOK / item["file"]).read_text(encoding="utf-8")
        all_text += "\n" + page
        if "\ufffd" in page or "\u00c3" in page or "\u00e2\u20ac" in page:
            fail(f"mojibake marker in {item['file']}")

    executive = (LOGBOOK / children[0]["file"]).read_text(encoding="utf-8")
    for required in (
        '"title":"Executive summary","pinned":true',
        '"type":"figure"',
        '"title":"Reproduction poster","pinned":true',
        "## Scope & cost",
        "This reproduction",
        "Full replication",
    ):
        if required not in executive:
            fail(f"executive summary missing {required}")
    poster = Path(LOGBOOK / children[0]["file"]).parent / "poster_embed.html"
    if not poster.is_file() or poster.stat().st_size < 8000:
        fail("poster_embed.html is missing or too small to be substantive")
    poster_text = poster.read_text(encoding="utf-8")
    if re.search(r"<(script|link)[^>]+(?:src|href)=['\"]https?://", poster_text, re.I):
        fail("poster has an external runtime dependency")
    if "aria-label" not in poster_text or "prefers-reduced-motion" not in poster_text:
        fail("poster accessibility hooks missing")

    claim_pages = [item for item in children if item["title"].startswith("Claim ")]
    required_sections = (
        "## Setup and method",
        "## Exact paper target",
        "## Reproduced result",
        "## Uncertainty and scale",
        "## Verdict:",
        "## Limitation",
        "## OpenResearch lineage",
        "## Artifact provenance",
    )
    for item in claim_pages:
        page = (LOGBOOK / item["file"]).read_text(encoding="utf-8")
        for section in required_sections:
            if section not in page:
                fail(f"{item['title']} missing {section}")
        for receipt in (
            "5d9e993d-65d3-41b1-845c-ec995c213e3e",
            "011ea9c7-db7c-4815-a732-f8054db7c59b",
            "sha256:4b4b1ef2383fecf4baa1565e7f9fbf2bc1b300b0b55dc67ba934f2bac5cf771a",
        ):
            if receipt not in page:
                fail(f"{item['title']} missing receipt {receipt}")

    if all_text.count(JOB_TOKEN) != 3:
        fail("HF Job token must appear only on empirical claims 4, 5, and 6")
    conclusion = (LOGBOOK / children[-1]["file"]).read_text(encoding="utf-8")
    if (
        conclusion.count(BUNDLE_TOKEN) != 1
        or '"type":"artifact"' not in conclusion
        or "## Verdict" in conclusion
    ):
        fail("Conclusion must contain only bundle status and rerun instructions")
    placeholders = set(re.findall(r"\{\{[A-Z0-9_]+\}\}", all_text))
    if placeholders != {JOB_TOKEN, BUNDLE_TOKEN}:
        fail(f"unexpected placeholders: {sorted(placeholders)}")

    forbidden = ("\u2014", "as an AI", "delve", "game-changing", "seamlessly", "robustly demonstrates")
    for token in forbidden:
        if token.lower() in all_text.lower():
            fail(f"forbidden prose token present: {token}")

    print("TRACKIO_LOGBOOK_VALID")
    print("pages=8 claims=6 pinned_summary=1 pinned_poster=1")
    print("external_placeholders=HF_JOB_URL:3 REPRO_BUNDLE:1")
    return 0


if __name__ == "__main__":
    try:
        raise SystemExit(main())
    except AssertionError as exc:
        print(f"TRACKIO_LOGBOOK_INVALID: {exc}", file=sys.stderr)
        raise SystemExit(1)