repro-bundle-v2/repro/audit_published_logbook.py

#!/usr/bin/env python3
"""Audit a published ICML-2026 reproduction logbook against the challenge checklist.

The challenge validator checks your *local* files. This checks what actually
landed on the Hub, which is where the interesting failures live: publish can
report success while the artifact bucket push failed, and a page can be present
but render as zero cells because of CRLF.

Checks, in the order the challenge lists them:

  1. Index    - "# Reproduction: <title>", a paper link, a Pages table, no cells
  2. Exec sum - pinned markdown titled "Executive summary", pinned FIRST
  3. Exec sum - pinned figure cell referencing poster_embed.html, pinned BELOW it
  4. Claims   - one page per claim, each with Hub/GitHub links
  5. Conclusn - an artifact cell + download/rerun prose, and no summary/poster
  6. Card     - title starts "Reproduction: ", tags icml2026-repro + paper-<orid>
  7. Bucket   - artifact cell resolves to huggingface.co/buckets/..., not
                trackio-artifact://
  8. Renders  - every page.md parses into >0 cells (the CRLF trap)
  9. Board    - the Space is discoverable by its tags

Usage:
    python audit_published_logbook.py <owner>/<space-name>
    python audit_published_logbook.py <owner>/<space> --token $HF_TOKEN

Only stdlib. Exit 0 if every hard check passes.
"""

from __future__ import annotations

import argparse
import json
import re
import sys
import urllib.error
import urllib.request

CELL_RE = re.compile(
    r"(^|\n)---\n<!-- trackio-cell\n([\s\S]*?)\n-->\n([\s\S]*?)"
    r"(?=\n---\n<!-- trackio-cell\n|\s*$)")

results: list[tuple[bool, str, str]] = []


def check(ok: bool, name: str, detail: str = "") -> bool:
    results.append((ok, name, detail))
    print(f"  [{'PASS' if ok else 'FAIL'}] {name}" + (f"  -- {detail}" if detail else ""))
    return ok


def get(url: str, token: str | None) -> str:
    req = urllib.request.Request(url)
    if token:
        req.add_header("Authorization", f"Bearer {token}")
    return urllib.request.urlopen(req, timeout=90).read().decode("utf-8", "replace")


def cells(md: str) -> list[dict]:
    out = []
    for m in CELL_RE.finditer(md):
        try:
            meta = json.loads(m.group(2))
        except json.JSONDecodeError:
            continue
        meta["_body"] = m.group(3)
        out.append(meta)
    return out


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("space", help="owner/space-name")
    ap.add_argument("--token", default=None, help="HF token (needed only if private)")
    args = ap.parse_args()

    if "/" not in args.space:
        sys.exit("space must be owner/name")
    owner, name = args.space.split("/", 1)
    raw = f"https://huggingface.co/spaces/{args.space}/raw/main"
    tok = args.token

    print(f"auditing https://huggingface.co/spaces/{args.space}\n")

    # --- card + tags -------------------------------------------------------
    print("Space card")
    try:
        readme = get(f"{raw}/README.md", tok)
    except urllib.error.HTTPError as e:
        sys.exit(f"cannot read Space README ({e.code}). Private? pass --token")
    m = re.search(r'^title:\s*"?(.+?)"?\s*$', readme, re.M)
    check(bool(m and m.group(1).startswith("Reproduction: ")),
          'card title starts "Reproduction: "', (m.group(1)[:60] if m else "no title"))
    check("icml2026-repro" in readme, "tag icml2026-repro")
    orid = re.search(r"paper-([A-Za-z0-9]{8,12})", readme)
    check(bool(orid), "tag paper-<openreview-id>", orid.group(0) if orid else "missing")
    check(name.startswith("repro-"), 'slug starts "repro-"', name[:50])
    check(not re.fullmatch(r"repro-[A-Za-z0-9]{8,12}", name),
          "slug is title-derived, not the OpenReview id")

    # --- structure ---------------------------------------------------------
    print("\nStructure")
    try:
        lb = json.loads(get(f"{raw}/logbook.json", tok))
    except Exception as e:                                 # noqa: BLE001
        sys.exit(f"cannot read logbook.json: {e}")
    root = lb["root"]
    slugs = [c["slug"] for c in root.get("children", [])]
    check(bool(slugs) and slugs[0] == "executive-summary", "first page is Executive summary")
    check(bool(slugs) and slugs[-1] == "conclusion", "last page is Conclusion")
    mids = slugs[1:-1]
    check(all(s.startswith("claim-") for s in mids),
          "middle pages are all claim-*", f"{len(mids)} claim page(s)")

    idx = get(f"{raw}/pages/index.md", tok)
    check(idx.lstrip().startswith("# Reproduction: "), "index heading")
    check(("openreview.net/forum?id=" in idx) or ("huggingface.co/papers/" in idx),
          "index links the paper")
    check("## Pages" in idx, "index has a Pages table")
    check("trackio-cell" not in idx, "index carries no cells")

    # --- per-page render sanity (the CRLF trap) ----------------------------
    print("\nEvery page parses into cells (catches CRLF corruption)")
    pages = {}
    for slug in slugs:
        path = f"{raw}/pages/{slug}/page.md"
        try:
            md = get(path, tok)
        except Exception:                                  # noqa: BLE001
            check(False, f"fetch {slug[:40]}")
            continue
        pages[slug] = md
        n = len(cells(md))
        crlf = "\r\n" in md
        check(n > 0 and not crlf, f"{slug[:44]}",
              f"{n} cell(s)" + (" -- CRLF PRESENT, will render as zero cells" if crlf else ""))

    # --- executive summary -------------------------------------------------
    print("\nExecutive summary")
    ex = cells(pages.get("executive-summary", ""))
    pinned = [c for c in ex if c.get("pinned")]
    check(bool(pinned) and pinned[0].get("type") == "markdown"
          and (pinned[0].get("title", "").lower() == "executive summary"),
          "pinned summary is first")
    check(any(c.get("type") == "figure" and c.get("pinned") for c in ex),
          "poster figure cell is pinned")
    check("poster_embed.html" in pages.get("executive-summary", ""),
          "poster references poster_embed.html")
    check("Scope & cost" in pages.get("executive-summary", ""), "Scope & cost table present")

    # --- conclusion --------------------------------------------------------
    print("\nConclusion")
    con = pages.get("conclusion", "")
    cc = cells(con)
    check(any(c.get("type") == "artifact" for c in cc), "artifact cell present")
    check("huggingface.co/buckets/" in con, "artifact resolves to a bucket URL")
    check("trackio-artifact://" not in con.replace("`trackio-artifact://`", ""),
          "no live trackio-artifact:// reference")
    check("Scope & cost" not in con, "no exec summary on Conclusion")
    check("posterly-embed" not in con, "no poster on Conclusion")

    # --- links -------------------------------------------------------------
    print("\nLinks across claim pages")
    allmd = "\n".join(pages.values())
    check(bool(re.search(r"https://huggingface\.co/(models|datasets|spaces|jobs|buckets)/", allmd))
          or bool(re.search(r"https://huggingface\.co/[\w.-]+/[\w.-]+", allmd)),
          "Hub assets linked")
    check("github.com/" in allmd, "GitHub repos linked")

    # --- board discovery ---------------------------------------------------
    print("\nBoard discovery")
    try:
        found = json.loads(get(
            f"https://huggingface.co/api/spaces?filter=icml2026-repro&author={owner}", tok))
        check(any(s.get("id") == args.space for s in found),
              "discoverable by icml2026-repro tag", f"{len(found)} space(s) for {owner}")
    except Exception as e:                                 # noqa: BLE001
        check(False, "board discovery query", str(e)[:60])

    failed = [r for r in results if not r[0]]
    print("\n" + "=" * 66)
    print(f"{len(results) - len(failed)}/{len(results)} checks passed")
    for _, n, d in failed:
        print(f"  FAIL: {n}" + (f" -- {d}" if d else ""))
    print("=" * 66)
    return 1 if failed else 0


if __name__ == "__main__":
    raise SystemExit(main())