bundle/v0/scripts/build_trackio_logbook.py

from __future__ import annotations

import json
from pathlib import Path


ROOT = Path(__file__).resolve().parents[1]
LOGBOOK = ROOT / ".trackio" / "logbook"
PAGES = LOGBOOK / "pages"
POSTER = ROOT / "logbook_assets" / "poster_embed.html"

PAPER_TITLE = (
    "Loss-Aware Distributionally Robust Optimization via Trainable Optimal "
    "Transport Ambiguity Sets"
)
PAPER_URL = "https://huggingface.co/papers/2509.12689"
OPENREVIEW_URL = "https://openreview.net/forum?id=K1EPPO9t2c"
JOB_TOKEN = "{{HF_JOB_URL_PENDING_AFTER_APPROVED_LAUNCH}}"

BASELINE = {
    "experiment": "5d9e993d-65d3-41b1-845c-ec995c213e3e",
    "run": "6d021de5-89c2-4b0d-a0b8-2997d5d3c1fa",
    "commit": "1bf139b9651fab1b9cf243e72ec340530d32eae2",
    "payload": "6ecedb11e6114ba0b44af7617489ebe89734c2512f9bfecf08b237a232b3a3e8",
}
CHILD = {
    "experiment": "011ea9c7-db7c-4815-a732-f8054db7c59b",
    "run": "b7320a54-2d8b-46a6-a89c-4f616eab936b",
    "commit": "656fbf32259324c2bae0117d314a2f817932c3b6",
    "payload": "44bd9190418b7309d25973cf418dee1cb877fb026dd1d8d40ceb033bf6d7bbf0",
}

AGGREGATE = "sha256:4b4b1ef2383fecf4baa1565e7f9fbf2bc1b300b0b55dc67ba934f2bac5cf771a"
RESULTS_SHA = "025f51a63fd75de100390b13374942cc5b57400d900b36c544777f9e67e0bd78"
MANIFEST = "sha256:ac1d3c6cb4b87b3b9deb430352aadc7a0bc07acb421c95f6b6858bee4a67be8d"
RECOVERY = "sha256:99c791f43a6423bf4e5f9c5710b92d1cfe85994e33758405ef2f0b15d22a9fbf"
ANALYSIS = "sha256:1ffc132991f8d970011d9c0aa49da3e04cc78d2b0363bab634cedd32714280fd"
CLAIMS_LOCK = "8c852634725aa2d678d87ec3517eca882a095af8770fcebe971547910943a046"
THEOREM = "5555a358cbe1040cd39c6637e2b57d9012318a0a9ae5d2a4a696e216466ae011"
HYPERGRADIENT = "85ba8545c76a780fdd68934686db1ce616e2a2162aa8c59e4860c9c802a28a43"


def cell(kind: str, cell_id: str, title: str, body: str, *, pinned: bool = False) -> str:
    metadata: dict[str, object] = {
        "type": kind,
        "id": cell_id,
        "created_at": "2026-07-20T09:30:22+00:00",
        "title": title,
    }
    if pinned:
        metadata["pinned"] = True
        metadata["pinned_at"] = "2026-07-20T09:30:22+00:00"
    return (
        "\n\n---\n<!-- trackio-cell\n"
        + json.dumps(metadata, ensure_ascii=False, separators=(",", ":"))
        + "\n-->\n"
        + body.strip()
        + "\n"
    )


def openresearch_receipt() -> str:
    return f"""## OpenResearch lineage

[OpenResearch](https://openresearch.sh) executed the same fixed command for a frozen baseline and one child boundary audit. Both runs validated all 14,000 rows locally at zero paid cost.

| Node | Experiment ID | Run ID | Commit | Result |
| --- | --- | --- | --- | --- |
| Baseline | `{BASELINE['experiment']}` | `{BASELINE['run']}` | `{BASELINE['commit']}` | done in 10 s |
| Leave-one-distribution-out child | `{CHILD['experiment']}` | `{CHILD['run']}` | `{CHILD['commit']}` | done in 5 s |

The OpenResearch repository is private during review, so these are immutable IDs rather than public repository links. Baseline payload: `{BASELINE['payload']}`. Child payload: `{CHILD['payload']}`."""


def provenance(*extra: str) -> str:
    items = [
        f"Frozen manifest: `{MANIFEST}`",
        f"Reconciled aggregate: `{AGGREGATE}`",
        f"14,000-row result file: `{RESULTS_SHA}`",
        f"Five-task recovery attestation: `{RECOVERY}`",
        f"Six-claim analysis: `{ANALYSIS}`",
    ]
    items.extend(extra)
    return "## Artifact provenance\n\n" + "\n".join(f"- {item}" for item in items)


CLAIM_BODIES = {
    1: f"""Can the ambiguity set itself be learned, instead of choosing one fixed ball before seeing the decision problem? This page tests the paper's central software object: an outer update to the transport geometry wrapped around a standard OT-DRO solve.

## Setup and method

We independently implemented the Gaussian portfolio and absolute-regression routes required by this claim. Each task freezes the dataset fingerprint, distribution seed, sample size, optimizer seed, lower-level solver contract, and out-of-sample evaluation. The broader matrix contains seven suites so the same implementation is also tested under discrete, mixture, high-dimensional, squared-loss, and coverage-ablation conditions.

## Exact paper target

The upper level must tune the ambiguity geometry parameter `theta`, while the lower level solves a standard OT-DRO problem. A full verification additionally requires the learning trajectories to reach a defensible terminal condition.

## Reproduced result

Both required bilevel routes executed. The reconciled matrix contains **14,000 validated tasks**, all lower-level solver checks were accepted, and exactly five controller exceptions were replaced by scientifically identical recovery outputs. However, **1,954 of the 5,500 claim-required tasks hit the preregistered 5,000-iteration cap**. Across the complete seven-suite matrix, 4,579 tasks were capped.

## Uncertainty and scale

The two required suites cover 50 portfolio distributions and 10 regression distributions, 10 dataset replicates per distribution, and 10 or 5 sample sizes respectively. This is execution evidence, not an estimate with a sampling interval. Capped trajectories are treated as right-censored.

## Verdict: partially verified

The bilevel formulation is implemented and exercised end to end, but a capped trajectory does not establish that the reported ambiguity geometry finished learning.

## Limitation

The 5,000-step ceiling preserves feasibility but is shorter than the paper's million-step maximum. We do not relabel a cap hit as convergence.

{openresearch_receipt()}

{provenance()}""",
    2: f"""A long optimization trace can look settled without satisfying a convergence theorem. This page separates numerical behavior from the mathematical statement in Theorem 5.1.

## Setup and method

We converted the theorem into a 12-item assumption and bridge audit. The audit checks the stated step-size regime, regularity assumptions, solution-map requirements, limiting argument, and whether the algorithm actually executed by the reproduction is the recurrence analyzed by the theorem.

## Exact paper target

Under the stated conditions, including square-summable step sizes, the proposed hypergradient descent procedure converges to a critical point of the bilevel problem.

## Reproduced result

No proof receipt establishes all theorem assumptions for the released instances. The executed trajectories are finite and capped, and the implementation used for the matrix is not certified as the ideal recurrence analyzed in the theorem.

## Uncertainty and scale

Sampling uncertainty is not the relevant quantity for a theorem. The evidence consists of 12 audited assumptions and logical bridges, plus finite trajectories that can illustrate behavior but cannot prove asymptotic convergence.

## Verdict: inconclusive

The numerical runs neither verify nor falsify Theorem 5.1. They fall outside the proof obligation.

## Limitation

A stronger verdict requires a hash-bound proof audit that closes every assumption and an execution contract demonstrably inside the theorem's scope.

{openresearch_receipt()}

{provenance(f'Theorem audit file: `{THEOREM}`')}""",
    3: f"""The difficult step is not solving the inner conic program. It is differentiating the outer decision through a solution map that may kink when the active set changes.

## Setup and method

Five pinned route configurations were checked with central finite differences at multiple step sizes: Gaussian portfolio exact, Gaussian portfolio moment class, empirical W1 portfolio, absolute regression, and squared regression. Each configuration also checks its active coverage-penalty component. Solver and transport residuals use a preregistered `1e-7` threshold.

## Exact paper target

Algorithm 1 must compute hypergradients through the nonsmooth conservative implicit function theorem without assuming that the conic solution map is continuously differentiable.

## Reproduced result

The component receipt reports all required route families passing at the chosen smooth evaluation points. The final six-claim analysis does not accept that component check as a paper-scale Algorithm 1 receipt because it does not bind active-set switching or the full nonsmooth execution path.

## Uncertainty and scale

This is a deterministic derivative check over five route families and several finite-difference step sizes, not a seeded performance estimate. Agreement at smooth points is necessary but insufficient for the nonsmooth claim.

## Verdict: inconclusive

Implemented gradients agree locally with finite differences. The conservative nonsmooth implicit-differentiation claim remains unverified.

## Limitation

The missing experiment must deliberately cross active-set boundaries, compare admissible generalized derivatives, and bind the exact Algorithm 1 formula to the paper source.

{openresearch_receipt()}

{provenance(f'Component hypergradient receipt: `{HYPERGRADIENT}`')}""",
    4: f"""When data are scarce, the shape of a transport ball should matter more. The paper's Figure 2 makes that intuition testable with one exact parameter contract.

## Setup and method

We swept portfolio sample sizes and measured both worst-case and out-of-sample relative improvement. Before scoring, the analysis compared the frozen matrix against the anchored Figure 2 tuple `k=2`, `J=30`, `n_b=20`, `gamma=0.05`, `beta=0.1` and the paper's relative-improvement formula.

## Exact paper target

Under that exact tuple and estimand, improvement from the learned ambiguity set over a fixed Wasserstein ball should increase as sample size decreases.

## Reproduced result

Only **3 of the 5 anchored setup fields** are bound by the frozen plan, and the exact Figure 2 estimand is not source-bound. The available sample-size slope is therefore reported only as an extension and receives no claim credit.

## Uncertainty and scale

The sensitivity sweep spans 50 Gaussian portfolio distributions, 10 replicates per distribution, and sample sizes 10 through 100. Distribution-first 95% deterministic bootstrap intervals use 10,000 replicates. Those intervals describe the extension, not the unmatched paper target.

## Verdict: inconclusive

The experiment is broad, but it is not the exact Figure 2 experiment.

## Limitation

This page needs a run with all five paper parameters and a source-audited formula before the trend can be scored.

## Hugging Face Job

**Pending insertion after launch:** `{JOB_TOKEN}`. The planned CPU Job is a preregistered scaled sensitivity audit across the Figure 2 sample-size axis. It does not bind the missing exact paper parameters and cannot by itself verify A4.

{openresearch_receipt()}

{provenance()}""",
    5: f"""A learned ambiguity set is useful only if it does not buy lower loss by abandoning the promised coverage. Figure 3 asks whether the true distribution still lies inside the learned set at least 90% of the time.

## Setup and method

For every portfolio sample size, coverage is averaged within each data-generating distribution and then aggregated across 50 distributions. This prevents the 10 replicates from masquerading as 500 independent distributions. The predeclared pass rule requires the 95% lower bound to reach 0.90 at every sample size.

## Exact paper target

The learned set preserves the `1 - beta = 0.90` coverage guarantee despite being shaped to reduce decision loss.

## Reproduced result

| Sample size | Coverage | 95% interval |
| ---: | ---: | ---: |
| 10 | 0.752 | [0.710, 0.792] |
| 20 | 0.798 | [0.762, 0.832] |
| 30 | 0.838 | [0.808, 0.866] |
| 40 | 0.822 | [0.784, 0.858] |
| 50 | 0.868 | [0.836, 0.898] |
| 60 | 0.852 | [0.820, 0.882] |
| 70 | 0.828 | [0.798, 0.858] |
| 80 | 0.842 | [0.812, 0.872] |
| 90 | 0.842 | [0.808, 0.876] |
| 100 | 0.834 | [0.796, 0.870] |

Every upper bound remains below 0.90. The pattern is adverse to the claim, but capped training trajectories prevent a terminal falsified verdict.

## Uncertainty and scale

Each row uses 50 independent distributions and 500 total dataset realizations. Intervals are distribution-first 95% deterministic percentile bootstrap intervals with 10,000 replicates.

## Verdict: inconclusive

The observed matrix misses the 90% target at every sample size. Because many learned geometries are right-censored, this is not yet a clean falsification of the paper's converged method.

## Limitation

The required follow-up must remove the training cap or establish that extending capped runs cannot restore coverage.

## Hugging Face Job

**Pending insertion after launch:** `{JOB_TOKEN}`. The planned CPU Job reruns a scaled coverage audit with the frozen implementation. It is substantive infrastructure evidence, but it does not remove the current training-cap limitation.

{openresearch_receipt()}

{provenance()}""",
    6: f"""Average improvement can hide a losing trial. Figure 5 uses stronger language: the learned set should produce lower regression loss consistently across ten independent trials.

## Setup and method

We evaluated the absolute-regression main suite at five sample sizes. Improvements are first averaged within each of the ten data-generating trials, then summarized across trials. The preregistered rule requires a positive 95% lower bound at every sample size and a positive mean for every individual trial.

## Exact paper target

All ten independent linear-regression trials yield less conservative decisions with lower average loss than the baseline OT-DRO ambiguity sets.

## Reproduced result

| Sample size | Relative OOS improvement | 95% interval |
| ---: | ---: | ---: |
| 10 | 0.0403 | [0.0231, 0.0575] |
| 20 | 0.0605 | [0.0358, 0.0857] |
| 30 | 0.0732 | [0.0450, 0.1028] |
| 40 | 0.0930 | [0.0556, 0.1304] |
| 50 | 0.1016 | [0.0618, 0.1410] |

The distribution-level mean is positive at all five sample sizes. The stricter all-ten-trials condition fails.

## Uncertainty and scale

Each row contains ten independent distributions and 100 total dataset realizations. Intervals are distribution-first 95% deterministic percentile bootstrap intervals with 10,000 replicates.

## Verdict: partially verified

The mean effect is consistently favorable across sample sizes, but the paper's stronger per-trial wording does not reproduce.

## Limitation

The matrix is capped, and the page does not reinterpret a positive grand mean as ten unanimous trial results.

## Hugging Face Job

**Pending insertion after launch:** `{JOB_TOKEN}`. The planned CPU Job covers all ten published regression trial families at scaled compute and records its public logs here. It does not replace the larger local matrix.

{openresearch_receipt()}

{provenance()}""",
}


def build() -> None:
    model = json.loads((LOGBOOK / "logbook.json").read_text(encoding="utf-8"))
    children = model["root"]["children"]
    claims = [item for item in children if item["title"].startswith("Claim ")]
    if len(claims) != 6:
        raise RuntimeError(f"expected six canonical claim pages, found {len(claims)}")

    index = (
        f"# Reproduction: {PAPER_TITLE}\n\n"
        f"[HF paper page]({PAPER_URL}) · [OpenReview]({OPENREVIEW_URL})\n\n"
        "## Pages\n\n| Page |\n| --- |\n"
        + "\n".join(f"| [{item['title']}](#/{item['slug']}) |" for item in children)
        + "\n"
    )
    (PAGES / "index.md").write_text(index, encoding="utf-8", newline="\n")

    summary_body = f"""**The central bilevel pipeline reproduces only in part.** We independently validated 14,000 task outputs with perfect solver acceptance, but 4,579 trajectories reached the 5,000-step cap, so convergence and downstream coverage cannot be promoted. Coverage stayed below the paper's 90% target at all ten sample sizes, while mean regression improvement was positive at all five sample sizes but not in every one of the ten trials. The full matrix used a local Intel Core Ultra 9 275HX CPU for 9 h 6 min wall time, 35.3 summed task-hours, and $0 paid compute; two clean OpenResearch evaluations added 15 seconds at $0.

## Scope & cost

| Item | This reproduction | Full replication |
| --- | --- | --- |
| Scope | 14,000 validated tasks across seven suites; 50 portfolio and 10 regression distributions; 10 replicates; five recovered controller exceptions | Exact anchored setup for every figure, uncapped or demonstrably converged optimization, and a public Hub rerun for empirical claims |
| Hardware | Local Intel Core Ultra 9 275HX CPU, four process workers, one BLAS thread per task | CPU is sufficient; a Hugging Face CPU Job supplies public execution provenance |
| Compute time | 9 h 6 min wall time; 35.3 summed task-hours; OpenResearch baseline + child 15 s | Unknown until the uncapped and claim-bound reruns finish |
| Cost | $0 paid compute so far | Pending exact Hub Job receipt; launch remains budget-capped |
| Outcome | A1 and A6 partially verified; A2 through A5 inconclusive | Full verdicts require the missing theorem, nonsmooth, exact Figure 2, convergence, and public-job evidence |"""
    summary_page = "# Executive summary\n" + cell(
        "markdown", "cell_ab6fce33e394", "Executive summary", summary_body, pinned=True
    )
    poster_html = POSTER.read_text(encoding="utf-8").strip()
    summary_page += cell(
        "figure",
        "cell_b8c145768d5a",
        "Reproduction poster",
        "````html\n<!-- poster_embed.html -->\n" + poster_html + "\n````",
        pinned=True,
    )
    executive = next(item for item in children if item["title"] == "Executive summary")
    executive_path = LOGBOOK / executive["file"]
    executive_path.write_text(summary_page, encoding="utf-8", newline="\n")
    (executive_path.parent / "poster_embed.html").write_text(
        poster_html + "\n", encoding="utf-8", newline="\n"
    )

    for number, item in enumerate(claims, start=1):
        title = item["title"]
        content = f"# {title}\n" + cell(
            "markdown", f"cell_claim_{number:02d}_evidence", title, CLAIM_BODIES[number]
        )
        (LOGBOOK / item["file"]).write_text(content, encoding="utf-8", newline="\n")

    conclusion = next(item for item in children if item["title"] == "Conclusion")
    artifact_body = """Reproduction bundle pending Trackio publication.

Local artifact name: `repro-loss-aware-distributionally-robust-optimization-via-trainable-optimal-transport-ambiguity/repro-bundle:v0`

Published Bucket URL: `{{REPRO_BUNDLE_ARTIFACT_CELL_INSERTED_BY_TRACKIO_PUBLISH}}`"""
    conclusion_body = """The reproduction bundle contains the independent implementation, frozen configs, environment locks, the reconciled 14,000-row aggregate, recovery attestation, theorem and hypergradient audits, raw plot data, OpenResearch receipts, and this deterministic logbook builder. It excludes secrets, virtual environments, and caches.

After downloading the bundle:

```bash
python -m pip install -r environment/requirements.lock
python -B scripts/validate_live_claims.py
python -B scripts/analyze_full_matrix.py \
  --manifest .openresearch/control/full-matrix-capped5000-v3/manifest.json \
  --reconciled-root .openresearch/artifacts/reconciled-full-matrix \
  --gradient-receipt .openresearch/artifacts/validation/hypergradient.json \
  --theorem-receipt configs/theorem-5.1-audit.json \
  --output reproduced-analysis.json
python -B presentation/validate_package.py
python -B scripts/validate_trackio_logbook.py
```

The artifact cell above will replace its token with a Hugging Face Bucket URL during publication. No private GitHub URL is presented as public code."""
    conclusion_page = "# Conclusion\n" + cell(
        "artifact", "cell_repro_bundle_pending", "Reproduction bundle", artifact_body
    )
    conclusion_page += cell(
        "markdown", "cell_70e28eff27d1", "Download and rerun", conclusion_body
    )
    (LOGBOOK / conclusion["file"]).write_text(
        conclusion_page, encoding="utf-8", newline="\n"
    )

    metadata_path = ROOT / ".trackio" / "metadata.json"
    metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
    metadata["emoji"] = "\U0001f9ed"
    metadata_path.write_text(
        json.dumps(metadata, indent=2, ensure_ascii=False) + "\n",
        encoding="utf-8",
        newline="\n",
    )
    model["emoji"] = "\U0001f9ed"
    (LOGBOOK / "logbook.json").write_text(
        json.dumps(model, indent=2, ensure_ascii=False) + "\n",
        encoding="utf-8",
        newline="\n",
    )


if __name__ == "__main__":
    build()