ctt measured rollout full validation 2026-07-03T01:27:41Z

workspace/scripts/build_ctt_rollout_comparison.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from scripts.eval_metrics import main as eval_metrics_main  # noqa: E402
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Aggregate measured CTT rollout runs.")
+    parser.add_argument(
+        "--run-glob",
+        default="runs/ctt_residual_rollout_val69_seed*",
+        help="Glob for run directories containing measured_candidates.json.",
+    )
+    parser.add_argument("--out-dir", type=Path, default=Path("runs/ctt_val_rollout_comparison"))
+    parser.add_argument("--k", type=int, default=8)
+    parser.add_argument("--bootstrap-samples", type=int, default=1000)
+    args = parser.parse_args(argv)
+
+    run_dirs = [
+        path
+        for path in sorted(Path().glob(args.run_glob))
+        if (path / "measured_candidates.json").exists()
+    ]
+    if not run_dirs:
+        raise SystemExit(f"no measured rollout runs found for glob {args.run_glob!r}")
+
+    combined_rows: list[dict[str, Any]] = []
+    payloads = []
+    for run_dir in run_dirs:
+        payload = json.loads((run_dir / "measured_candidates.json").read_text())
+        payloads.append(payload)
+        train_seed = _seed_from_path(run_dir)
+        for row in payload.get("rows", []):
+            item = dict(row)
+            item["train_seed"] = train_seed
+            item["seed"] = f"{row.get('seed', 'unknown')}/train{train_seed}"
+            combined_rows.append(item)
+
+    out_dir = args.out_dir
+    out_dir.mkdir(parents=True, exist_ok=True)
+    combined = {
+        "report_type": "ctt_measured_rollout_comparison",
+        "schema_version": 1,
+        "k": args.k,
+        "run_dirs": [str(path) for path in run_dirs],
+        "train_seeds": [_seed_from_path(path) for path in run_dirs],
+        "num_rows": len(combined_rows),
+        "source_content_hash": _first(payloads, "source_content_hash"),
+        "target_content_hash": _first(payloads, "target_content_hash"),
+        "target_split_hash": _first(payloads, "target_split_hash"),
+        "rows": combined_rows,
+    }
+    combined_path = out_dir / "combined_measured_candidates.json"
+    combined_path.write_text(json.dumps(combined, indent=2, sort_keys=True) + "\n")
+    eval_metrics_main(
+        [
+            "--input",
+            str(combined_path),
+            "--out-dir",
+            str(out_dir / "measured_metrics"),
+            "--mode",
+            "measured",
+            "--k",
+            str(args.k),
+            "--bootstrap-samples",
+            str(args.bootstrap_samples),
+        ]
+    )
+    metrics = json.loads((out_dir / "measured_metrics" / "metrics.json").read_text())
+    (out_dir / "metrics.json").write_text(json.dumps(_summary_payload(combined, metrics), indent=2, sort_keys=True) + "\n")
+    (out_dir / "table.tex").write_text((out_dir / "measured_metrics" / "table.tex").read_text())
+    (out_dir / "report.md").write_text(_report(combined, metrics) + "\n")
+    (out_dir / "command.txt").write_text(
+        "python scripts/build_ctt_rollout_comparison.py " + " ".join(sys.argv[1:]) + "\n"
+    )
+    (out_dir / "git_hash.txt").write_text(_run(["git", "rev-parse", "HEAD"]) + "\n")
+    print(json.dumps({"out_dir": str(out_dir), "runs": len(run_dirs), "rows": len(combined_rows)}, indent=2))
+    return 0
+
+
+def _summary_payload(combined: dict[str, Any], metrics: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "report_type": "ctt_measured_rollout_comparison",
+        "k": combined["k"],
+        "run_dirs": combined["run_dirs"],
+        "train_seeds": combined["train_seeds"],
+        "num_rows": combined["num_rows"],
+        "summary": metrics.get("summary", {}),
+        "source_content_hash": combined.get("source_content_hash"),
+        "target_content_hash": combined.get("target_content_hash"),
+        "target_split_hash": combined.get("target_split_hash"),
+    }
+
+
+def _report(combined: dict[str, Any], metrics: dict[str, Any]) -> str:
+    summary = metrics.get("summary", {})
+    lines = [
+        "# CTT Validation Measured Rollout Comparison",
+        "",
+        f"Runs: `{len(combined['run_dirs'])}`",
+        f"Rows: `{combined['num_rows']}`",
+        f"K: `{combined['k']}`",
+        "",
+        "| Metric | N | Micro mean | 95% CI |",
+        "| --- | ---: | ---: | ---: |",
+    ]
+    for name, payload in sorted(summary.items()):
+        micro = payload.get("micro", {})
+        lines.append(
+            f"| {name} | {micro.get('n', 0)} | {_fmt(micro.get('mean'))} | "
+            f"[{_fmt(micro.get('low'))}, {_fmt(micro.get('high'))}] |"
+        )
+    lines.append("")
+    lines.append("These are measured generated-candidate rollouts, not PPTC proxies.")
+    lines.append("")
+    lines.append("Run dirs:")
+    for run_dir in combined["run_dirs"]:
+        lines.append(f"- `{run_dir}`")
+    return "\n".join(lines)
+
+
+def _seed_from_path(path: Path) -> str:
+    match = re.search(r"seed(\d+)", path.name)
+    return match.group(1) if match else path.name
+
+
+def _first(payloads: list[dict[str, Any]], key: str) -> Any:
+    for payload in payloads:
+        value = payload.get(key)
+        if value:
+            return value
+    return None
+
+
+def _fmt(value: Any) -> str:
+    if not isinstance(value, (int, float)) or not math.isfinite(float(value)):
+        return "n/a"
+    return f"{float(value):.4f}"
+
+
+def _run(command: list[str]) -> str:
+    try:
+        return subprocess.check_output(command, cwd=PROJECT_ROOT, text=True).strip()
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return ""
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())