auto-sync 2026-07-04T06:15:18Z workspace (part 4)
Browse files
workspace/tests/test_selector_diagnostic_sweep.py
CHANGED
|
@@ -85,3 +85,44 @@ def test_selector_diagnostic_sweep_exports_best_by_family(tmp_path: Path, monkey
|
|
| 85 |
"table.tex",
|
| 86 |
):
|
| 87 |
assert (out_dir / filename).exists()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
"table.tex",
|
| 86 |
):
|
| 87 |
assert (out_dir / filename).exists()
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def test_selector_diagnostic_sweep_keeps_utility_energy_as_own_family(
|
| 91 |
+
tmp_path: Path, monkeypatch
|
| 92 |
+
) -> None:
|
| 93 |
+
monkeypatch.chdir(tmp_path)
|
| 94 |
+
utility = tmp_path / "runs/ctt_dominance_utility_energy_val_to_test_seed2/metrics.json"
|
| 95 |
+
utility.parent.mkdir(parents=True, exist_ok=True)
|
| 96 |
+
utility.write_text(
|
| 97 |
+
json.dumps(
|
| 98 |
+
{
|
| 99 |
+
"report_type": "dominance_calibrated_selector_eval",
|
| 100 |
+
"k": 16,
|
| 101 |
+
"score_source": "checkpoint",
|
| 102 |
+
"tau_mode": "auto",
|
| 103 |
+
"data_hash": "data-utility",
|
| 104 |
+
"split_hash": "split-utility",
|
| 105 |
+
"eval_summary": {
|
| 106 |
+
"base_success": 0.29,
|
| 107 |
+
"selected_success": 0.28,
|
| 108 |
+
"proposal_oracle_success": 0.57,
|
| 109 |
+
"coverage": 0.17,
|
| 110 |
+
"success_support_gap": 0.20,
|
| 111 |
+
"success_selector_gap": 0.31,
|
| 112 |
+
"outcome_ptr": 0.55,
|
| 113 |
+
"pairwise_causal_calibration_ece": 0.02,
|
| 114 |
+
},
|
| 115 |
+
}
|
| 116 |
+
)
|
| 117 |
+
+ "\n"
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
out_dir = tmp_path / "runs/ctt_selector_diagnostic_sweep"
|
| 121 |
+
assert sweep.main(["--metrics", str(utility), "--out-dir", str(out_dir)]) == 0
|
| 122 |
+
|
| 123 |
+
payload = json.loads((out_dir / "metrics.json").read_text())
|
| 124 |
+
by_family = {row["family"]: row for row in payload["best_by_family"]}
|
| 125 |
+
assert by_family["K16 env_clip utility-energy"]["seed"] == "2"
|
| 126 |
+
assert "checkpoint utility energy" in by_family["K16 env_clip utility-energy"]["selector"]
|
| 127 |
+
by_seed = json.loads((out_dir / "metrics_by_seed.json").read_text())
|
| 128 |
+
assert by_seed["2"]["selected_success"] == 0.28
|