expand CTT proxy comparison metrics

scripts/build_ctt_proxy_comparison.py

@@ -43,6 +43,11 @@ def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(description="Build CTT validation proxy comparison table.")
     parser.add_argument("--out-dir", type=Path, default=Path("runs/ctt_val_proxy_comparison"))
     parser.add_argument("--safety-slack", type=float, default=0.01)
+    parser.add_argument(
+        "--no-markdown-report",
+        action="store_true",
+        help="Do not write report.md; the persistent prose summary lives in README.md.",
+    )
     args = parser.parse_args(argv)
 
     rows = [
@@ -74,6 +79,7 @@ def main(argv: list[str] | None = None) -> int:
     out_dir.mkdir(parents=True, exist_ok=True)
     payload = {
         "report_type": "ctt_val_proxy_comparison",
+        "schema_version": 2,
         "baseline": "local_atlas",
         "safety_slack": args.safety_slack,
         "rows": rows,
@@ -81,10 +87,21 @@ def main(argv: list[str] | None = None) -> int:
         "split_hash": _first(rows, "split_hash"),
     }
     (out_dir / "metrics.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
-    (out_dir / "metrics_by_task.json").write_text("{}\n")
-    (out_dir / "metrics_by_seed.json").write_text("{}\n")
+    (out_dir / "metrics_by_task.json").write_text(
+        json.dumps(_grouped_metrics(DEFAULT_RUNS, group_key="task_id"), indent=2, sort_keys=True)
+        + "\n"
+    )
+    (out_dir / "metrics_by_seed.json").write_text(
+        json.dumps(_grouped_metrics(DEFAULT_RUNS, group_key="seed"), indent=2, sort_keys=True)
+        + "\n"
+    )
     (out_dir / "table.tex").write_text(_table(rows) + "\n")
-    (out_dir / "report.md").write_text(_report(rows, args.safety_slack) + "\n")
+    _write_report_artifact(
+        out_dir,
+        rows,
+        safety_slack=args.safety_slack,
+        no_markdown_report=args.no_markdown_report,
+    )
     (out_dir / "train.log").write_text("comparison artifact; no training\n")
     (out_dir / "eval.log").write_text(
         "\n".join(f"{row['method']}: {row['run_path']}" for row in rows) + "\n"
@@ -113,7 +130,10 @@ def _row(method: str, run_dirs: list[Path]) -> dict[str, Any]:
         "negative_near_0p20": _mean_across(summaries, "negative_near_at_16_thr_0p20"),
         "negative_near_0p40": _mean_across(summaries, "negative_near_at_16_thr_0p40"),
         "mean_positive_distance": _mean_across(summaries, "mean_positive_distance_at_16"),
+        "mean_negative_distance": _mean_across(summaries, "mean_negative_distance_at_16"),
+        "pos_closer_than_neg": _mean_across(summaries, "pos_closer_than_neg_at_16"),
         "candidate_diversity": _mean_across(summaries, "candidate_diversity_at_16"),
+        "collapse_rate": _mean_across(summaries, "collapse_rate_at_16"),
         "proxy_support_distance": _mean_across(summaries, "proxy_support_distance_at_16"),
         "data_hash": payloads[0].get("data_hash"),
         "split_hash": payloads[0].get("target_split_hash", payloads[0].get("split_hash")),
@@ -132,17 +152,19 @@ def _mean_across(summaries: list[dict[str, Any]], key: str) -> float:
 def _table(rows: list[dict[str, Any]]) -> str:
     lines = [
         "% Auto-generated by scripts/build_ctt_proxy_comparison.py",
-        "\\begin{tabular}{lrrrrrrrc}",
+        "\\begin{tabular}{lrrrrrrrrrrc}",
         "\\toprule",
-        "Method & Seeds & Rows & PPTC@0.20 & PPTC@0.40 & Neg@0.20 & MeanPosDist & Diversity & Gate \\\\",
+        "Method & Seeds & Rows & PPTC@0.20 & PPTC@0.40 & Neg@0.20 & Neg@0.40 & Pos<Neg & MeanPos & MeanNeg & Collapse & Gate \\\\",
         "\\midrule",
     ]
     for row in rows:
         lines.append(
             f"{_latex_escape(row['method'])} & {row['train_seeds']} & {row['num_rows']} & "
             f"{_fmt(row['pptc_0p20'])} & {_fmt(row['pptc_0p40'])} & "
-            f"{_fmt(row['negative_near_0p20'])} & {_fmt(row['mean_positive_distance'])} & "
-            f"{_fmt(row['candidate_diversity'])} & {_gate(row)} \\\\"
+            f"{_fmt(row['negative_near_0p20'])} & {_fmt(row['negative_near_0p40'])} & "
+            f"{_fmt(row['pos_closer_than_neg'])} & {_fmt(row['mean_positive_distance'])} & "
+            f"{_fmt(row['mean_negative_distance'])} & {_fmt(row['collapse_rate'])} & "
+            f"{_gate(row)} \\\\"
         )
     lines.extend(["\\bottomrule", "\\end{tabular}"])
     return "\n".join(lines)
@@ -154,15 +176,17 @@ def _report(rows: list[dict[str, Any]], safety_slack: float) -> str:
         "",
         f"Safety slack over local-atlas NegativeNear@0.20: `{safety_slack:.4f}`",
         "",
-        "| Method | Seeds | Rows | PPTC@0.20 | PPTC@0.40 | Neg@0.20 | MeanPosDist | Diversity | Gate | Run |",
-        "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |",
+        "| Method | Seeds | Rows | PPTC@0.20 | PPTC@0.40 | Neg@0.20 | Neg@0.40 | Pos<Neg | MeanPos | MeanNeg | Diversity | Collapse | Gate | Run |",
+        "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |",
     ]
     for row in rows:
         lines.append(
             f"| {row['method']} | {row['train_seeds']} | {row['num_rows']} | "
             f"{_fmt(row['pptc_0p20'])} | "
             f"{_fmt(row['pptc_0p40'])} | {_fmt(row['negative_near_0p20'])} | "
-            f"{_fmt(row['mean_positive_distance'])} | {_fmt(row['candidate_diversity'])} | "
+            f"{_fmt(row['negative_near_0p40'])} | {_fmt(row['pos_closer_than_neg'])} | "
+            f"{_fmt(row['mean_positive_distance'])} | {_fmt(row['mean_negative_distance'])} | "
+            f"{_fmt(row['candidate_diversity'])} | {_fmt(row['collapse_rate'])} | "
             f"{_gate(row)} | `{row['run_path']}` |"
         )
     lines.append("")
@@ -170,6 +194,66 @@ def _report(rows: list[dict[str, Any]], safety_slack: float) -> str:
     return "\n".join(lines)
 
 
+def _write_report_artifact(
+    out_dir: Path,
+    rows: list[dict[str, Any]],
+    *,
+    safety_slack: float,
+    no_markdown_report: bool,
+) -> None:
+    report_path = out_dir / "report.md"
+    if no_markdown_report:
+        if report_path.exists():
+            report_path.unlink()
+        return
+    report_path.write_text(_report(rows, safety_slack) + "\n")
+
+
+def _grouped_metrics(
+    run_specs: list[tuple[str, list[Path]]],
+    *,
+    group_key: str,
+) -> dict[str, dict[str, dict[str, float]]]:
+    output: dict[str, dict[str, dict[str, float]]] = {}
+    for method, run_dirs in run_specs:
+        if not run_dirs or not all((run_dir / "metrics.json").exists() for run_dir in run_dirs):
+            continue
+        rows: list[dict[str, Any]] = []
+        for run_dir in run_dirs:
+            payload = json.loads((run_dir / "metrics.json").read_text())
+            rows.extend(payload.get("rows", []))
+        if not rows:
+            continue
+        method_groups: dict[str, dict[str, list[float]]] = {}
+        for row in rows:
+            group = str(row.get(group_key, "unknown"))
+            metrics = method_groups.setdefault(group, {})
+            for source_key, target_key in _row_metric_key_map().items():
+                value = row.get(source_key)
+                if isinstance(value, (int, float)) and math.isfinite(float(value)):
+                    metrics.setdefault(target_key, []).append(float(value))
+        output[method] = {
+            group: {name: sum(values) / len(values) for name, values in sorted(metrics.items())}
+            for group, metrics in sorted(method_groups.items())
+        }
+    return output
+
+
+def _row_metric_key_map() -> dict[str, str]:
+    return {
+        "pptc_at_16_thr_0p20": "pptc_0p20",
+        "pptc_at_16_thr_0p40": "pptc_0p40",
+        "negative_near_at_16_thr_0p20": "negative_near_0p20",
+        "negative_near_at_16_thr_0p40": "negative_near_0p40",
+        "mean_positive_distance_at_16": "mean_positive_distance",
+        "mean_negative_distance_at_16": "mean_negative_distance",
+        "pos_closer_than_neg_at_16": "pos_closer_than_neg",
+        "candidate_diversity_at_16": "candidate_diversity",
+        "collapse_rate_at_16": "collapse_rate",
+        "proxy_support_distance_at_16": "proxy_support_distance",
+    }
+
+
 def _fmt(value: Any) -> str:
     if not isinstance(value, (int, float)) or not math.isfinite(float(value)):
         return "n/a"