auto-sync 2026-07-04T04:28:21Z workspace (part 7)

workspace/scripts/build_ctt_rollout_comparison.py

@@ -35,6 +35,11 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--out-dir", type=Path, default=Path("runs/ctt_val_rollout_comparison"))
     parser.add_argument("--k", type=int, default=8)
     parser.add_argument("--bootstrap-samples", type=int, default=1000)
+    parser.add_argument(
+        "--no-markdown-report",
+        action="store_true",
+        help="Do not write report.md; persistent prose lives in README.md.",
+    )
     args = parser.parse_args(argv)
 
     run_dirs = [
@@ -75,24 +80,29 @@ def main(argv: list[str] | None = None) -> int:
     }
     combined_path = out_dir / "combined_measured_candidates.json"
     combined_path.write_text(json.dumps(combined, indent=2, sort_keys=True) + "\n")
-    eval_metrics_main(
-        [
-            "--input",
-            str(combined_path),
-            "--out-dir",
-            str(out_dir / "measured_metrics"),
-            "--mode",
-            "measured",
-            "--k",
-            str(args.k),
-            "--bootstrap-samples",
-            str(args.bootstrap_samples),
-        ]
-    )
+    metric_args = [
+        "--input",
+        str(combined_path),
+        "--out-dir",
+        str(out_dir / "measured_metrics"),
+        "--mode",
+        "measured",
+        "--k",
+        str(args.k),
+        "--bootstrap-samples",
+        str(args.bootstrap_samples),
+    ]
+    if args.no_markdown_report:
+        metric_args.append("--no-markdown-report")
+    eval_metrics_main(metric_args)
     metrics = json.loads((out_dir / "measured_metrics" / "metrics.json").read_text())
     (out_dir / "metrics.json").write_text(json.dumps(_summary_payload(combined, metrics), indent=2, sort_keys=True) + "\n")
     (out_dir / "table.tex").write_text((out_dir / "measured_metrics" / "table.tex").read_text())
-    (out_dir / "report.md").write_text(_report(combined, metrics) + "\n")
+    report_path = out_dir / "report.md"
+    if args.no_markdown_report:
+        report_path.unlink(missing_ok=True)
+    else:
+        report_path.write_text(_report(combined, metrics) + "\n")
     for filename in ("metrics_by_task.json", "metrics_by_seed.json"):
         shutil.copyfile(out_dir / "measured_metrics" / filename, out_dir / filename)
     (out_dir / "command.txt").write_text(

workspace/scripts/eval_ctt_generated_rollout.py

@@ -130,6 +130,11 @@ def main(argv: list[str] | None = None) -> int:
     )
     parser.add_argument("--skip-metrics", action="store_true")
     parser.add_argument("--bootstrap-samples", type=int, default=200)
+    parser.add_argument(
+        "--no-markdown-report",
+        action="store_true",
+        help="Do not write report.md files; persistent prose lives in README.md.",
+    )
     args = parser.parse_args(argv)
 
     if args.k <= 0:
@@ -296,24 +301,29 @@ def main(argv: list[str] | None = None) -> int:
     }
     measured_path = out_dir / "measured_candidates.json"
     measured_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
-    (out_dir / "report.md").write_text(_report(payload) + "\n")
+    report_path = out_dir / "report.md"
+    if args.no_markdown_report:
+        report_path.unlink(missing_ok=True)
+    else:
+        report_path.write_text(_report(payload) + "\n")
 
     metrics_dir = out_dir / "measured_metrics"
     if not args.skip_metrics:
-        eval_metrics_main(
-            [
-                "--input",
-                str(measured_path),
-                "--out-dir",
-                str(metrics_dir),
-                "--mode",
-                "measured",
-                "--k",
-                str(args.k),
-                "--bootstrap-samples",
-                str(args.bootstrap_samples),
-            ]
-        )
+        metric_args = [
+            "--input",
+            str(measured_path),
+            "--out-dir",
+            str(metrics_dir),
+            "--mode",
+            "measured",
+            "--k",
+            str(args.k),
+            "--bootstrap-samples",
+            str(args.bootstrap_samples),
+        ]
+        if args.no_markdown_report:
+            metric_args.append("--no-markdown-report")
+        eval_metrics_main(metric_args)
     _write_required_artifacts(
         out_dir,
         payload,

workspace/scripts/eval_ctt_rollout.py

@@ -25,24 +25,30 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--k", type=int, default=16)
     parser.add_argument("--epsilon", type=float, default=0.0)
     parser.add_argument("--bootstrap-samples", type=int, default=1000)
+    parser.add_argument(
+        "--no-markdown-report",
+        action="store_true",
+        help="Do not write report.md in the metric output directory.",
+    )
     args = parser.parse_args(argv)
 
-    return eval_metrics_main(
-        [
-            "--input",
-            str(args.measured_candidates),
-            "--out-dir",
-            str(args.out_dir),
-            "--mode",
-            "measured",
-            "--k",
-            str(args.k),
-            "--epsilon",
-            str(args.epsilon),
-            "--bootstrap-samples",
-            str(args.bootstrap_samples),
-        ]
-    )
+    forwarded = [
+        "--input",
+        str(args.measured_candidates),
+        "--out-dir",
+        str(args.out_dir),
+        "--mode",
+        "measured",
+        "--k",
+        str(args.k),
+        "--epsilon",
+        str(args.epsilon),
+        "--bootstrap-samples",
+        str(args.bootstrap_samples),
+    ]
+    if args.no_markdown_report:
+        forwarded.append("--no-markdown-report")
+    return eval_metrics_main(forwarded)
 
 
 if __name__ == "__main__":

workspace/scripts/eval_dominance_selector.py

@@ -69,6 +69,11 @@ def main(argv: list[str] | None = None) -> int:
     )
     parser.add_argument("--k", type=int, default=8)
     parser.add_argument("--bootstrap-samples", type=int, default=1000)
+    parser.add_argument(
+        "--no-markdown-report",
+        action="store_true",
+        help="Do not write report.md; persistent prose lives in README.md.",
+    )
     args = parser.parse_args(argv)
 
     if not 0.0 < args.alpha < 1.0:
@@ -190,7 +195,11 @@ def main(argv: list[str] | None = None) -> int:
         + "\n"
     )
     (out_dir / "table.tex").write_text(_table(metrics) + "\n")
-    (out_dir / "report.md").write_text(_report(metrics) + "\n")
+    report_path = out_dir / "report.md"
+    if args.no_markdown_report:
+        report_path.unlink(missing_ok=True)
+    else:
+        report_path.write_text(_report(metrics) + "\n")
     (out_dir / "train.log").write_text(
         "fit conformal residual quantile and tau on calibration measured rows only\n"
         f"calibration_input={args.calibration_input}\n"

workspace/scripts/eval_metrics.py

@@ -2,8 +2,10 @@
 from __future__ import annotations
 
 import argparse
+import hashlib
 import json
 import math
+import subprocess
 import sys
 from collections import defaultdict
 from pathlib import Path
@@ -52,6 +54,11 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--thresholds", default="0.20,0.40")
     parser.add_argument("--bootstrap-samples", type=int, default=1000)
     parser.add_argument("--confidence", type=float, default=0.95)
+    parser.add_argument(
+        "--no-markdown-report",
+        action="store_true",
+        help="Do not write report.md; use this when README.md is the only persistent Markdown file.",
+    )
     args = parser.parse_args(argv)
 
     if args.k <= 0:
@@ -119,7 +126,12 @@ def main(argv: list[str] | None = None) -> int:
         + "\n"
     )
     (out_dir / "table.tex").write_text(_latex_table(summary) + "\n")
-    (out_dir / "report.md").write_text(_markdown_report(args.mode, args.k, summary) + "\n")
+    _write_run_metadata(out_dir, args, payload, metric_names)
+    report_path = out_dir / "report.md"
+    if args.no_markdown_report:
+        report_path.unlink(missing_ok=True)
+    else:
+        report_path.write_text(_markdown_report(args.mode, args.k, summary) + "\n")
     print(json.dumps({"out_dir": str(out_dir), "num_rows": len(metric_rows)}, indent=2))
     return 0
 
@@ -435,6 +447,98 @@ def _group_means(
     return output
 
 
+def _write_run_metadata(
+    out_dir: Path,
+    args: argparse.Namespace,
+    input_payload: Any,
+    metric_names: list[str],
+) -> None:
+    data_hash = _payload_hash(input_payload)
+    split_hash = _extract_hash(
+        input_payload,
+        (
+            "split_hash",
+            "target_split_hash",
+            "eval_target_split_hash",
+            "selector_split_hash",
+        ),
+    )
+    if split_hash is None:
+        split_hash = data_hash
+
+    (out_dir / "config.yaml").write_text(
+        "\n".join(
+            [
+                f"input: {args.input}",
+                f"mode: {args.mode}",
+                f"k: {args.k}",
+                f"epsilon: {args.epsilon}",
+                f"thresholds: {args.thresholds}",
+                f"bootstrap_samples: {args.bootstrap_samples}",
+                f"confidence: {args.confidence}",
+                f"no_markdown_report: {bool(args.no_markdown_report)}",
+                "metric_names:",
+                *[f"  - {name}" for name in metric_names],
+            ]
+        )
+        + "\n"
+    )
+    (out_dir / "command.txt").write_text(
+        "python scripts/eval_metrics.py " + " ".join(sys.argv[1:]) + "\n"
+    )
+    (out_dir / "git_hash.txt").write_text(_git_hash() + "\n")
+    (out_dir / "data_hash.txt").write_text(data_hash + "\n")
+    (out_dir / "split_hash.txt").write_text(split_hash + "\n")
+    (out_dir / "train.log").write_text("metric evaluation artifact; no training\n")
+    (out_dir / "eval.log").write_text(
+        "\n".join(
+            [
+                f"input={args.input}",
+                f"mode={args.mode}",
+                f"k={args.k}",
+                f"num_metrics={len(metric_names)}",
+                f"markdown_report_written={not bool(args.no_markdown_report)}",
+            ]
+        )
+        + "\n"
+    )
+
+
+def _payload_hash(payload: Any) -> str:
+    blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode()
+    return hashlib.sha256(blob).hexdigest()
+
+
+def _extract_hash(payload: Any, keys: tuple[str, ...]) -> str | None:
+    if isinstance(payload, dict):
+        for key in keys:
+            value = payload.get(key)
+            if isinstance(value, str) and value.strip():
+                return value.strip()
+        for value in payload.values():
+            nested = _extract_hash(value, keys)
+            if nested is not None:
+                return nested
+    elif isinstance(payload, list):
+        for value in payload:
+            nested = _extract_hash(value, keys)
+            if nested is not None:
+                return nested
+    return None
+
+
+def _git_hash() -> str:
+    try:
+        return subprocess.check_output(
+            ["git", "rev-parse", "HEAD"],
+            cwd=PROJECT_ROOT,
+            text=True,
+            stderr=subprocess.DEVNULL,
+        ).strip()
+    except (OSError, subprocess.CalledProcessError):
+        return "unknown"
+
+
 def _latex_table(summary: dict[str, Any]) -> str:
     lines = [
         "% Auto-generated by scripts/eval_metrics.py",