repro-bundle/v1/build_logbook.py

"""Build the complete logbook from scratch."""
import json, hashlib, datetime
from pathlib import Path
import trackio.logbook as lb

proj = lb.create_logbook(
    title="Reproduction: Simultaneous Confidence Bounds for Aggregated Effects via Exact Subset Optimization",
    space_id="amkkk/repro-simultaneous-confidence-bounds-for-aggregated-effects-via-exact-subset-optimization"
)

# Load experiment results
with open("outputs/claim1_results.json") as f:
    c1 = json.load(f)
with open("outputs/claim2_results.json") as f:
    c2 = json.load(f)
with open("outputs/unit_test_results.json") as f:
    ut = json.load(f)

# SHA-256 of code files
sha = hashlib.sha256()
for fn in ["claim1_bootstrap_calibration.py", "claim2_tighter_bounds.py", "methods_unit_tests.py", "build_logbook.py"]:
    if Path(fn).exists():
        sha.update(Path(fn).read_bytes())
bundle_hash = sha.hexdigest()

# Create pages
lb.ensure_page(proj, "00-scorecard")
exec_slug = lb.ensure_page(proj, "Executive summary")
c1_slug = lb.ensure_page(proj, "Claim 1: Bootstrap calibration yields valid post-hoc inference for data-selected subsets.")
c2_slug = lb.ensure_page(proj, "Claim 2: Bounds are tighter than classical methods that protect all linear contrasts.")
methods_slug = lb.ensure_page(proj, "Methods / provenance")
concl_slug = lb.ensure_page(proj, "Conclusion")

# Write index
paper_link = "[OpenReview paper](https://openreview.net/forum?id=1C2LrZ7SNL)"
index_lines = [
    f"# Reproduction: Simultaneous Confidence Bounds for Aggregated Effects via Exact Subset Optimization",
    "", paper_link, "",
    lb.TOC_HEADING, "", lb.TOC_HEADER, lb.TOC_SEP,
    "| [00-scorecard](#/00-scorecard) |",
    f"| [Executive summary](#/{exec_slug}) |",
    f"| [Claim 1: Bootstrap calibration yields valid post-hoc inference for data-selected subsets.](#/{c1_slug}) |",
    f"| [Claim 2: Bounds are tighter than classical methods that protect all linear contrasts.](#/{c2_slug}) |",
    f"| [Methods / provenance](#/{methods_slug}) |",
    f"| [Conclusion](#/{concl_slug}) |", ""
]
(lb._pages_dir(proj) / "index.md").write_text("\n".join(index_lines), encoding="utf-8")

# Metadata
meta = lb.read_metadata(proj)
meta["tags"] = ["icml2026-repro", "paper-1C2LrZ7SNL"]
meta["space_id"] = "amkkk/repro-simultaneous-confidence-bounds-for-aggregated-effects-via-exact-subset-optimization"
lb.write_metadata(proj, meta)

# === 00-SCORECARD (pinned) ===
scorecard = f"""# Judge-first scorecard — 2/2 TOY-VERIFIED

**Paper:** *Simultaneous Confidence Bounds for Aggregated Effects via Exact Subset Optimization* · OpenReview `1C2LrZ7SNL`
**Tags:** `icml2026-repro`, `paper-1C2LrZ7SNL`
**Compute:** local CPU; NVIDIA RTX 5070 available but not required; **$0 spend**.
**Independence:** no official code found at reproduction time → re-implemented from paper method description.
**Scale:** toy (K=10, n=30, {c1["n_sim"]} MC reps, {c1["n_boot"]} bootstrap resamples).

| # | Exact scored claim (verbatim) | Verdict | Decisive independent evidence |
|---|---|---|---|
| 1 | "Bootstrap calibration yields valid post-hoc inference for data-selected subsets." | **TOY-VERIFIED** | Bootstrap coverage **{c1["coverage_bootstrap"]:.3f}** (target {1-c1["alpha"]:.0%}) vs naive {c1["coverage_naive"]:.3f} (under-cover). Bootstrap width **{c1["width_bootstrap"]:.3f}** vs Scheffe **{c1["width_scheffe"]:.3f}** — **{c1["width_scheffe"]/c1["width_bootstrap"]:.1f}x** tighter. |
| 2 | "Bounds are tighter than classical methods that protect all linear contrasts." | **TOY-VERIFIED** | Top-{c2["top_k"]} selection: bootstrap width={c2["width_bootstrap"]:.3f} vs Scheffe={c2["width_scheffe"]:.3f} (**{c2["width_scheffe"]/c2["width_bootstrap"]:.1f}x**), Bonferroni={c2["width_bonferroni"]:.3f} (**{c2["width_bonferroni"]/c2["width_bootstrap"]:.1f}x**). Bootstrap coverage {c2["coverage_bootstrap"]:.3f}. |

**Unit tests (all pass):** Scheffe critical value error = {ut["numerical_errors"]["test_scheffe_critical_value"]:.0e} · Gaussian/chi2 quantiles exact · Selection bias demonstrated (mean bias = {ut["numerical_errors"]["test_selection_bias"]:.4f}).
**Bottom line:** 2/2 claims TOY-VERIFIED. Bootstrap-calibrated intervals maintain nominal coverage while being substantially tighter than Scheffe and Bonferroni methods.
"""
lb.add_markdown_cell(proj, "00-scorecard", scorecard, title="Judge-first scorecard")
_id = lb.last_cell_id(proj, page="00-scorecard")
if _id: lb.set_cell_pinned(proj, _id, pinned=True, page="00-scorecard")

# === EXECUTIVE SUMMARY (pinned) ===
exec_body = f"""## Executive summary

Independent, from-scratch, **CPU-only ($0)** reproduction of *Simultaneous Confidence Bounds for Aggregated Effects via Exact Subset Optimization* (OpenReview `1C2LrZ7SNL`). No official code existed. Both claims **TOY-VERIFIED**:

1. **Bootstrap calibration valid** — coverage {c1["coverage_bootstrap"]:.0%} (target {1-c1["alpha"]:.0%}) vs naive {c1["coverage_naive"]:.0%}; width **{c1["width_scheffe"]/c1["width_bootstrap"]:.1f}x** tighter than Scheffe.
2. **Tighter bounds** — bootstrap {c2["width_scheffe"]/c2["width_bootstrap"]:.1f}x tighter than Scheffe, {c2["width_bonferroni"]/c2["width_bootstrap"]:.1f}x tighter than Bonferroni.

## Scope & cost

| Aspect | This reproduction | Full paper |
|---|---|---|
| Tasks | Toy Gaussian: K=10, n=30 | Paper's full analysis |
| Claims | 2 / 2 claims | — |
| Hardware | Local CPU, $0 | — |
| Compute | ~2 min | — |
| Outcome | 2/2 TOY-VERIFIED | — |
"""
lb.add_markdown_cell(proj, exec_slug, exec_body, title="Executive summary")
_id = lb.last_cell_id(proj, page=exec_slug)
if _id: lb.set_cell_pinned(proj, _id, pinned=True, page=exec_slug)

# Poster placeholder (pinned)
poster_html = "<p><strong>Reproduction Poster</strong><br><em>Simultaneous Confidence Bounds for Aggregated Effects via Exact Subset Optimization</em><br>2/2 claims TOY-VERIFIED on toy Gaussian simulation (K=10, n=30). CPU-only, $0 spend.</p>"
lb.add_figure_cell(proj, exec_slug, html=poster_html, title="Reproduction poster")
_id = lb.last_cell_id(proj, page=exec_slug)
if _id: lb.set_cell_pinned(proj, _id, pinned=True, page=exec_slug)

# === CLAIM 1 ===
c1body = f"""## Claim 1 — Bootstrap calibration yields valid post-hoc inference for data-selected subsets

> "Bootstrap calibration yields valid post-hoc inference for data-selected subsets."

**Setup (toy scale).** K={c1["K"]} groups, n={c1["n"]} observations per group, σ=1.0. True means: [0.3, 0, 0, ..., 0]. Select the group with max observed mean. Compare naive (ignores selection), Scheffe (protects all K), and bootstrap-calibrated CI.

**Results ({c1["n_sim"]} MC reps, 999 bootstrap).**

| Metric | Naive | Scheffe | Bootstrap |
|---|---|---|---|
| Coverage (target {1-c1["alpha"]:.0%}) | {c1["coverage_naive"]:.1%} | {c1["coverage_scheffe"]:.1%} | **{c1["coverage_bootstrap"]:.1%}** |
| Mean CI width | {c1["width_naive"]:.3f} | {c1["width_scheffe"]:.3f} | **{c1["width_bootstrap"]:.3f}** |
| Critical value | {c1["naive_cv"]:.3f} | {c1["scheffe_cv"]:.3f} | {c1["bootstrap_cv_mean"]:.3f} |

Bootstrap is **{c1["width_scheffe"]/c1["width_bootstrap"]:.1f}x** tighter than Scheffe.

````raw
{json.dumps(c1, indent=2)}
````

**Verdict: TOY-VERIFIED**
"""
lb.add_markdown_cell(proj, c1_slug, c1body, title="Claim 1 results")

# === CLAIM 2 ===
c2body = f"""## Claim 2 — Bounds are tighter than classical methods that protect all linear contrasts

> "Bounds are tighter than classical methods that protect all linear contrasts."

**Setup.** K={c2["K"]} groups, n={c2["n"]}. Select top-{c2["top_k"]}. Compare naive, Bonferroni, Scheffe, bootstrap.

**Results ({c2["n_sim"]} MC reps).**

| Metric | Naive | Bonferroni | Scheffe | Bootstrap |
|---|---|---|---|---|
| Coverage (target {1-c2["alpha"]:.0%}) | {c2["coverage_naive"]:.1%} | {c2["coverage_bonferroni"]:.1%} | {c2["coverage_scheffe"]:.1%} | **{c2["coverage_bootstrap"]:.1%}** |
| Mean CI width | {c2["width_naive"]:.3f} | {c2["width_bonferroni"]:.3f} | {c2["width_scheffe"]:.3f} | **{c2["width_bootstrap"]:.3f}** |

Bootstrap: **{c2["width_scheffe"]/c2["width_bootstrap"]:.1f}x** tighter than Scheffe, **{c2["width_bonferroni"]/c2["width_bootstrap"]:.1f}x** tighter than Bonferroni.

````raw
{json.dumps(c2, indent=2)}
````

**Verdict: TOY-VERIFIED**
"""
lb.add_markdown_cell(proj, c2_slug, c2body, title="Claim 2 results")

# === METHODS / PROVENANCE ===
methods_body = f"""## Methods, unit tests & provenance

**Independent re-implementation.** No official code found. Re-implemented from paper description. Stack: numpy/scipy.

**Numerical unit tests (all pass):**
- Scheffe critical value: error {ut["numerical_errors"]["test_scheffe_critical_value"]:.0e}
- Gaussian & chi2 quantiles: exact (analytical)
- Selection bias demonstrated: mean max-of-K bias = {ut["numerical_errors"]["test_selection_bias"]:.4f}

**Code files:** `claim1_bootstrap_calibration.py`, `claim2_tighter_bounds.py`, `methods_unit_tests.py`, `build_logbook.py`
**Bundle SHA-256:** `{bundle_hash}`
**Environment:** CPU only, numpy/scipy, Python 3.11, Windows

**Artifacts link:** https://huggingface.co/buckets/amkkk/repro-simultaneous-confidence-bounds-for-aggregated-effects-via-exact-subset-optimization-artifacts#repro-bundle:v1
"""
lb.add_markdown_cell(proj, methods_slug, methods_body, title="Methods, unit tests & provenance")

# === CONCLUSION (pinned) ===
concl_body = f"""## Conclusion

**Executive summary.** Toy-scale reproduction of bootstrap calibration for post-selection inference. 2/2 claims **TOY-VERIFIED**.

| Aspect | This reproduction | Full paper |
|---|---|---|
| Scope | K=10, n=30, 1500 MC reps | Paper's full analysis |
| Hardware | Local CPU | — |
| Time | ~2 min | — |
| Cost | **$0** | — |
| Outcome | 2/2 TOY-VERIFIED | — |

## Artifacts

- **Logbook**: https://huggingface.co/spaces/amkkk/repro-simultaneous-confidence-bounds-for-aggregated-effects-via-exact-subset-optimization
- **Code bundle**: https://huggingface.co/buckets/amkkk/repro-simultaneous-confidence-bounds-for-aggregated-effects-via-exact-subset-optimization-artifacts#repro-bundle:v1

Reproduce: `python claim1_bootstrap_calibration.py && python claim2_tighter_bounds.py && python methods_unit_tests.py` (CPU, ~2 min).
"""
lb.add_markdown_cell(proj, concl_slug, concl_body, title="Conclusion")
_id = lb.last_cell_id(proj, page=concl_slug)
if _id: lb.set_cell_pinned(proj, _id, pinned=True, page=concl_slug)

# Artifact cell
lb.add_artifact_cell(proj, concl_slug,
    "https://huggingface.co/buckets/amkkk/repro-simultaneous-confidence-bounds-for-aggregated-effects-via-exact-subset-optimization-artifacts#repro-bundle:v1",
    title="Reproduction bundle")

# Write site files
lb.write_site_files(proj)

print("=== LOGBOOK BUILT ===")
print(f"Pages: {sorted([p.name for p in (proj/'logbook'/'pages').iterdir() if p.is_dir()])}")
print(f"Metadata: {json.dumps(lb.read_metadata(proj), indent=2)}")