manual-outcome-gate-sync 2026-07-04T04:15:33Z workspace (part 5)

workspace/tests/test_ctt_outcome_gate.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from scripts import build_ctt_outcome_gate as outcome_gate
+
+
+def _micro(mean: float) -> dict:
+    return {"micro": {"mean": mean, "low": mean, "high": mean}}
+
+
+def test_ctt_outcome_gate_records_acceptance_failures_without_markdown(tmp_path) -> None:
+    rollout_dir = tmp_path / "rollout"
+    selector_dir = tmp_path / "selector"
+    rollout_dir.mkdir()
+    selector_dir.mkdir()
+    rollout_metrics = rollout_dir / "metrics.json"
+    selector_metrics = selector_dir / "metrics.json"
+    rollout_metrics.write_text(
+        json.dumps(
+            {
+                "data_hash": "rollout-data",
+                "split_hash": "rollout-split",
+                "train_seeds": ["0", "1", "2"],
+                "summary": {
+                    "outcome_ptr_at_16": _micro(0.55),
+                    "proposal_oracle_success_at_16": _micro(0.57),
+                    "success_support_gap_at_16": _micro(0.20),
+                    "generated_unsafe_rate_known_at_16": _micro(0.0),
+                    "selected_unsafe_known_at_16": _micro(0.0),
+                    "base_unsafe_known": _micro(0.0),
+                },
+            }
+        )
+        + "\n"
+    )
+    selector_metrics.write_text(
+        json.dumps(
+            {
+                "data_hash": "selector-data",
+                "split_hash": "selector-split",
+                "eval_summary": {
+                    "base_success": 0.29,
+                    "selected_success": 0.35,
+                    "proposal_oracle_success": 0.57,
+                    "hidden_chart_oracle_success": 0.73,
+                    "success_support_gap": 0.20,
+                    "success_selector_gap": 0.24,
+                    "pairwise_causal_calibration_ece": 0.01,
+                },
+                "summary": {
+                    "selected_success": _micro(0.35),
+                    "proposal_oracle_success": _micro(0.57),
+                    "success_selector_gap": _micro(0.24),
+                },
+                "rows": [{"train_seed": "0"}, {"train_seed": "1"}, {"train_seed": "2"}],
+            }
+        )
+        + "\n"
+    )
+    (rollout_dir / "metrics_by_task.json").write_text('{"rollout": {}}\n')
+    (selector_dir / "metrics_by_task.json").write_text('{"selector": {}}\n')
+    (rollout_dir / "metrics_by_seed.json").write_text('{"rollout": {}}\n')
+    (selector_dir / "metrics_by_seed.json").write_text('{"selector": {}}\n')
+
+    out_dir = tmp_path / "gate"
+    assert (
+        outcome_gate.main(
+            [
+                "--rollout-metrics",
+                str(rollout_metrics),
+                "--selector-metrics",
+                str(selector_metrics),
+                "--out-dir",
+                str(out_dir),
+            ]
+        )
+        == 0
+    )
+
+    metrics = json.loads((out_dir / "metrics.json").read_text())
+    gates = {gate["name"]: gate for gate in metrics["gates"]}
+    by_task = json.loads((out_dir / "metrics_by_task.json").read_text())
+
+    assert metrics["overall_pass"] is False
+    assert metrics["status"] == "diagnostic_not_method_success"
+    assert gates["proposal oracle success"]["pass"] is True
+    assert gates["true OutcomePTR@16 vs V0"]["pass"] is True
+    assert gates["selected clean success"]["pass"] is False
+    assert gates["success support gap"]["pass"] is False
+    assert gates["success selector gap"]["pass"] is False
+    assert gates["three train seeds"]["pass"] is True
+    assert by_task["selector_summary"] == {"selector": {}}
+    assert by_task["rollout_summary"] == {"rollout": {}}
+    assert "selected clean success" in (out_dir / "table.tex").read_text()
+    assert not (out_dir / "report.md").exists()

workspace/tests/test_ctt_proxy_comparison.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from scripts import build_ctt_proxy_comparison as comparison
+
+
+def _write_proxy_run(
+    run_dir: Path,
+    *,
+    pptc20: float,
+    pptc40: float,
+    neg20: float,
+    neg40: float,
+    pos_closer: float,
+    mean_pos: float,
+    mean_neg: float,
+    diversity: float,
+    collapse: float,
+) -> None:
+    run_dir.mkdir(parents=True)
+    row = {
+        "chart_id": "c0",
+        "task_id": "PickCube-v1",
+        "seed": "0",
+        "pptc_at_16_thr_0p20": pptc20,
+        "pptc_at_16_thr_0p40": pptc40,
+        "negative_near_at_16_thr_0p20": neg20,
+        "negative_near_at_16_thr_0p40": neg40,
+        "pos_closer_than_neg_at_16": pos_closer,
+        "mean_positive_distance_at_16": mean_pos,
+        "mean_negative_distance_at_16": mean_neg,
+        "candidate_diversity_at_16": diversity,
+        "collapse_rate_at_16": collapse,
+        "proxy_support_distance_at_16": mean_pos,
+    }
+    summary = {
+        key: {"micro": {"mean": value}}
+        for key, value in row.items()
+        if isinstance(value, float)
+    }
+    (run_dir / "metrics.json").write_text(
+        json.dumps(
+            {
+                "num_rows": 1,
+                "summary": summary,
+                "rows": [row],
+                "data_hash": "data",
+                "target_split_hash": "split",
+            },
+            indent=2,
+        )
+        + "\n"
+    )
+
+
+def test_ctt_proxy_comparison_writes_full_proxy_gate_without_markdown(
+    tmp_path, monkeypatch
+) -> None:
+    local = tmp_path / "local_atlas"
+    ctt = tmp_path / "ctt_residual"
+    _write_proxy_run(
+        local,
+        pptc20=0.4,
+        pptc40=0.7,
+        neg20=0.03,
+        neg40=0.2,
+        pos_closer=0.6,
+        mean_pos=0.7,
+        mean_neg=0.8,
+        diversity=0.9,
+        collapse=0.05,
+    )
+    _write_proxy_run(
+        ctt,
+        pptc20=0.2,
+        pptc40=0.6,
+        neg20=0.035,
+        neg40=0.25,
+        pos_closer=0.8,
+        mean_pos=0.4,
+        mean_neg=0.5,
+        diversity=0.3,
+        collapse=0.06,
+    )
+    monkeypatch.setattr(
+        comparison,
+        "DEFAULT_RUNS",
+        [("local_atlas", [local]), ("ctt_residual", [ctt])],
+    )
+
+    out_dir = tmp_path / "comparison"
+    assert comparison.main(["--out-dir", str(out_dir), "--no-markdown-report"]) == 0
+
+    metrics = json.loads((out_dir / "metrics.json").read_text())
+    ctt_row = next(row for row in metrics["rows"] if row["method"] == "ctt_residual")
+    by_task = json.loads((out_dir / "metrics_by_task.json").read_text())
+    by_seed = json.loads((out_dir / "metrics_by_seed.json").read_text())
+
+    assert ctt_row["proxy_gate_pass"] is True
+    assert ctt_row["pos_closer_than_neg"] == 0.8
+    assert ctt_row["mean_negative_distance"] == 0.5
+    assert ctt_row["collapse_rate"] == 0.06
+    assert by_task["ctt_residual"]["PickCube-v1"]["pos_closer_than_neg"] == 0.8
+    assert by_seed["ctt_residual"]["0"]["mean_negative_distance"] == 0.5
+    assert "Pos<Neg" in (out_dir / "table.tex").read_text()
+    assert not (out_dir / "report.md").exists()