support markdown-free CTT summary

scripts/summarize_ctt_runs.py

@@ -27,6 +27,7 @@ COLUMNS = [
     "pptc_0p20",
     "pptc_0p40",
     "negative_near_0p20",
+    "unsafe_execution",
     "calibration_ece",
     "seeds",
     "ci",
@@ -38,9 +39,15 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--run-root", type=Path, default=Path("runs"))
     parser.add_argument("--out-csv", type=Path, default=Path("runs/summary_ctt.csv"))
     parser.add_argument("--out-md", type=Path, default=Path("runs/summary_ctt.md"))
+    parser.add_argument(
+        "--no-markdown-report",
+        action="store_true",
+        help="Do not write --out-md; the persistent prose summary lives in README.md.",
+    )
     args = parser.parse_args(argv)
 
     rows = []
+    seen_metric_paths: set[Path] = set()
     reproduce = args.run_root / "reproduce_v0" / "metrics.json"
     if reproduce.exists():
         rows.extend(_reproduce_rows(reproduce))
@@ -50,9 +57,11 @@ def main(argv: list[str] | None = None) -> int:
             "ctt_val_proxy_comparison",
             "dominance_calibrated_selector_eval",
             "learned_dominance_selector_eval",
+            "nonlinear_dominance_selector_eval",
         }:
             continue
         rows.append(_ctt_row(metrics_path))
+        seen_metric_paths.add(metrics_path)
     for metrics_path in sorted(args.run_root.glob("*memory*_val_proxy/metrics.json")):
         rows.append(_positive_memory_row(metrics_path))
     local_atlas = args.run_root / "local_atlas_val_proxy" / "metrics.json"
@@ -64,12 +73,30 @@ def main(argv: list[str] | None = None) -> int:
     utility = args.run_root / "utility_energy_smoke" / "metrics.json"
     if utility.exists():
         rows.append(_utility_row(utility))
-    for metrics_path in sorted(args.run_root.glob("ctt_dominance*/metrics.json")):
-        rows.append(_dominance_row(metrics_path))
-    for metrics_path in sorted(args.run_root.glob("ctt_learned_dominance*/metrics.json")):
-        rows.append(_dominance_row(metrics_path))
+    for pattern in ("ctt_dominance*/metrics.json", "ctt*_dominance*/metrics.json"):
+        for metrics_path in sorted(args.run_root.glob(pattern)):
+            if metrics_path in seen_metric_paths:
+                continue
+            report_type = _report_type(metrics_path)
+            if report_type not in {
+                "dominance_calibrated_selector_eval",
+                "learned_dominance_selector_eval",
+                "nonlinear_dominance_selector_eval",
+            }:
+                continue
+            rows.append(_dominance_row(metrics_path))
+            seen_metric_paths.add(metrics_path)
+    for pattern in ("ctt_learned_dominance*/metrics.json", "ctt*_learned_dominance*/metrics.json"):
+        for metrics_path in sorted(args.run_root.glob(pattern)):
+            if metrics_path in seen_metric_paths:
+                continue
+            rows.append(_dominance_row(metrics_path))
+            seen_metric_paths.add(metrics_path)
     for metrics_path in sorted(args.run_root.glob("ctt*_nonlinear_dominance*/metrics.json")):
+        if metrics_path in seen_metric_paths:
+            continue
         rows.append(_dominance_row(metrics_path))
+        seen_metric_paths.add(metrics_path)
 
     args.out_csv.parent.mkdir(parents=True, exist_ok=True)
     with args.out_csv.open("w", newline="") as handle:
@@ -77,8 +104,14 @@ def main(argv: list[str] | None = None) -> int:
         writer.writeheader()
         for row in rows:
             writer.writerow({column: row.get(column, "n/a") for column in COLUMNS})
-    args.out_md.write_text(_markdown(rows) + "\n")
-    print(json.dumps({"csv": str(args.out_csv), "md": str(args.out_md), "rows": len(rows)}, indent=2))
+    if args.no_markdown_report:
+        if args.out_md.exists():
+            args.out_md.unlink()
+        md_path: str | None = None
+    else:
+        args.out_md.write_text(_markdown(rows) + "\n")
+        md_path = str(args.out_md)
+    print(json.dumps({"csv": str(args.out_csv), "md": md_path, "rows": len(rows)}, indent=2))
     return 0
 
 
@@ -113,6 +146,7 @@ def _reproduce_rows(path: Path) -> list[dict[str, str]]:
             "negative_near_0p20": _fmt(
                 proxy.get("negative_near_at_0p20", proxy.get("negative_near_at_16_thr_0p2"))
             ),
+            "unsafe_execution": "n/a",
             "calibration_ece": "n/a",
             "seeds": "3",
             "ci": "see run",
@@ -144,6 +178,7 @@ def _ctt_row(path: Path) -> dict[str, str]:
         "pptc_0p20": _summary_mean(summary, "pptc_at_16_thr_0p20"),
         "pptc_0p40": _summary_mean(summary, "pptc_at_16_thr_0p40"),
         "negative_near_0p20": _summary_mean(summary, "negative_near_at_16_thr_0p20"),
+        "unsafe_execution": "n/a",
         "calibration_ece": "n/a",
         "seeds": _seed_count(path.parent),
         "ci": "see run",
@@ -172,6 +207,7 @@ def _positive_memory_row(path: Path) -> dict[str, str]:
         "pptc_0p20": _summary_mean(summary, "pptc_at_16_thr_0p20"),
         "pptc_0p40": _summary_mean(summary, "pptc_at_16_thr_0p40"),
         "negative_near_0p20": _summary_mean(summary, "negative_near_at_16_thr_0p20"),
+        "unsafe_execution": "n/a",
         "calibration_ece": "n/a",
         "seeds": _seed_count(path.parent),
         "ci": "see run",
@@ -204,6 +240,7 @@ def _comparison_rows(path: Path) -> list[dict[str, str]]:
                 "pptc_0p20": _fmt(item.get("pptc_0p20")),
                 "pptc_0p40": _fmt(item.get("pptc_0p40")),
                 "negative_near_0p20": _fmt(item.get("negative_near_0p20")),
+                "unsafe_execution": "n/a",
                 "calibration_ece": "n/a",
                 "seeds": str(item.get("train_seeds", "unknown")),
                 "ci": "aggregate over run means",
@@ -232,6 +269,7 @@ def _utility_row(path: Path) -> dict[str, str]:
         "pptc_0p20": "n/a",
         "pptc_0p40": "n/a",
         "negative_near_0p20": "n/a",
+        "unsafe_execution": "n/a",
         "calibration_ece": _summary_mean(data.get("summary", {}), "pairwise_causal_calibration_ece"),
         "seeds": "train split",
         "ci": "see run",
@@ -248,6 +286,11 @@ def _dominance_row(path: Path) -> dict[str, str]:
     proposal_oracle = _as_float(summary.get("proposal_oracle_success"))
     success_support_gap = _as_float(summary.get("success_support_gap"))
     success_selector_gap = _as_float(summary.get("success_selector_gap"))
+    ece = _first_float(
+        _dominance_pairwise_ece(data, "eval"),
+        summary.get("pairwise_causal_calibration_ece"),
+        _metric_mean(metric_summary, "pairwise_causal_calibration_ece"),
+    )
     seed_count = _dominance_seed_count(data)
     target = data.get("target")
     feature_set = data.get("feature_set")
@@ -285,12 +328,23 @@ def _dominance_row(path: Path) -> dict[str, str]:
         "pptc_0p20": "n/a",
         "pptc_0p40": "n/a",
         "negative_near_0p20": "n/a",
-        "calibration_ece": "n/a",
+        "unsafe_execution": _fmt(summary.get("unsafe_execution_known")),
+        "calibration_ece": _fmt(ece),
         "seeds": str(seed_count),
         "ci": "see run",
     }
 
 
+def _dominance_pairwise_ece(data: dict[str, Any], split: str) -> float | None:
+    payload = data.get("pairwise_causal_calibration", {})
+    if not isinstance(payload, dict):
+        return None
+    split_payload = payload.get(split, {})
+    if not isinstance(split_payload, dict):
+        return None
+    return _as_float(split_payload.get("ece"))
+
+
 def _dominance_method_name(path: Path, data: dict[str, Any]) -> str:
     if "nonlinear" in path.parent.name:
         target = data.get("target") or "positive_margin"
@@ -365,6 +419,7 @@ def _measured_ctt_row(path: Path, data: dict[str, Any]) -> dict[str, str]:
         "pptc_0p20": "n/a",
         "pptc_0p40": "n/a",
         "negative_near_0p20": "n/a",
+        "unsafe_execution": _summary_mean(summary, f"selected_unsafe_known_at_{k}"),
         "calibration_ece": _fmt(ece),
         "seeds": str(seed_count),
         "ci": "see measured_metrics",
@@ -493,8 +548,8 @@ def _markdown(rows: list[dict[str, str]]) -> str:
     lines = [
         "# CTT Summary",
         "",
-        "| Method | Status | Base | Selected | Proposal oracle | Hidden oracle | Utility support gap | Utility selector gap | Success support gap | Success selector gap | OutcomePTR | PPTC@0.20 | PPTC@0.40 | NegativeNear@0.20 | Calibration ECE | Run |",
-        "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |",
+        "| Method | Status | Base | Selected | Proposal oracle | Hidden oracle | Utility support gap | Utility selector gap | Success support gap | Success selector gap | OutcomePTR | PPTC@0.20 | PPTC@0.40 | NegativeNear@0.20 | Unsafe exec. | Calibration ECE | Run |",
+        "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |",
     ]
     for row in rows:
         lines.append(
@@ -502,7 +557,7 @@ def _markdown(rows: list[dict[str, str]]) -> str:
             "{proposal_oracle} | {hidden_oracle} | {support_gap} | {selector_gap} | "
             "{success_support_gap} | {success_selector_gap} | {outcome_ptr} | "
             "{pptc_0p20} | {pptc_0p40} | "
-            "{negative_near_0p20} | {calibration_ece} | `{run_path}` |".format(**row)
+            "{negative_near_0p20} | {unsafe_execution} | {calibration_ece} | `{run_path}` |".format(**row)
         )
     lines.append("")
     lines.append(