test CTT proxy comparison artifact

tests/test_ctt_proxy_comparison.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from scripts import build_ctt_proxy_comparison as comparison
+
+
+def _write_proxy_run(
+    run_dir: Path,
+    *,
+    pptc20: float,
+    pptc40: float,
+    neg20: float,
+    neg40: float,
+    pos_closer: float,
+    mean_pos: float,
+    mean_neg: float,
+    diversity: float,
+    collapse: float,
+) -> None:
+    run_dir.mkdir(parents=True)
+    row = {
+        "chart_id": "c0",
+        "task_id": "PickCube-v1",
+        "seed": "0",
+        "pptc_at_16_thr_0p20": pptc20,
+        "pptc_at_16_thr_0p40": pptc40,
+        "negative_near_at_16_thr_0p20": neg20,
+        "negative_near_at_16_thr_0p40": neg40,
+        "pos_closer_than_neg_at_16": pos_closer,
+        "mean_positive_distance_at_16": mean_pos,
+        "mean_negative_distance_at_16": mean_neg,
+        "candidate_diversity_at_16": diversity,
+        "collapse_rate_at_16": collapse,
+        "proxy_support_distance_at_16": mean_pos,
+    }
+    summary = {
+        key: {"micro": {"mean": value}}
+        for key, value in row.items()
+        if isinstance(value, float)
+    }
+    (run_dir / "metrics.json").write_text(
+        json.dumps(
+            {
+                "num_rows": 1,
+                "summary": summary,
+                "rows": [row],
+                "data_hash": "data",
+                "target_split_hash": "split",
+            },
+            indent=2,
+        )
+        + "\n"
+    )
+
+
+def test_ctt_proxy_comparison_writes_full_proxy_gate_without_markdown(
+    tmp_path, monkeypatch
+) -> None:
+    local = tmp_path / "local_atlas"
+    ctt = tmp_path / "ctt_residual"
+    _write_proxy_run(
+        local,
+        pptc20=0.4,
+        pptc40=0.7,
+        neg20=0.03,
+        neg40=0.2,
+        pos_closer=0.6,
+        mean_pos=0.7,
+        mean_neg=0.8,
+        diversity=0.9,
+        collapse=0.05,
+    )
+    _write_proxy_run(
+        ctt,
+        pptc20=0.2,
+        pptc40=0.6,
+        neg20=0.035,
+        neg40=0.25,
+        pos_closer=0.8,
+        mean_pos=0.4,
+        mean_neg=0.5,
+        diversity=0.3,
+        collapse=0.06,
+    )
+    monkeypatch.setattr(
+        comparison,
+        "DEFAULT_RUNS",
+        [("local_atlas", [local]), ("ctt_residual", [ctt])],
+    )
+
+    out_dir = tmp_path / "comparison"
+    assert comparison.main(["--out-dir", str(out_dir), "--no-markdown-report"]) == 0
+
+    metrics = json.loads((out_dir / "metrics.json").read_text())
+    ctt_row = next(row for row in metrics["rows"] if row["method"] == "ctt_residual")
+    by_task = json.loads((out_dir / "metrics_by_task.json").read_text())
+    by_seed = json.loads((out_dir / "metrics_by_seed.json").read_text())
+
+    assert ctt_row["proxy_gate_pass"] is True
+    assert ctt_row["pos_closer_than_neg"] == 0.8
+    assert ctt_row["mean_negative_distance"] == 0.5
+    assert ctt_row["collapse_rate"] == 0.06
+    assert by_task["ctt_residual"]["PickCube-v1"]["pos_closer_than_neg"] == 0.8
+    assert by_seed["ctt_residual"]["0"]["mean_negative_distance"] == 0.5
+    assert "Pos<Neg" in (out_dir / "table.tex").read_text()
+    assert not (out_dir / "report.md").exists()