File size: 3,097 Bytes
c86c41e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 | from __future__ import annotations
import json
from pathlib import Path
from scripts import build_ctt_proxy_comparison as comparison
def _write_proxy_run(
run_dir: Path,
*,
pptc20: float,
pptc40: float,
neg20: float,
neg40: float,
pos_closer: float,
mean_pos: float,
mean_neg: float,
diversity: float,
collapse: float,
) -> None:
run_dir.mkdir(parents=True)
row = {
"chart_id": "c0",
"task_id": "PickCube-v1",
"seed": "0",
"pptc_at_16_thr_0p20": pptc20,
"pptc_at_16_thr_0p40": pptc40,
"negative_near_at_16_thr_0p20": neg20,
"negative_near_at_16_thr_0p40": neg40,
"pos_closer_than_neg_at_16": pos_closer,
"mean_positive_distance_at_16": mean_pos,
"mean_negative_distance_at_16": mean_neg,
"candidate_diversity_at_16": diversity,
"collapse_rate_at_16": collapse,
"proxy_support_distance_at_16": mean_pos,
}
summary = {
key: {"micro": {"mean": value}}
for key, value in row.items()
if isinstance(value, float)
}
(run_dir / "metrics.json").write_text(
json.dumps(
{
"num_rows": 1,
"summary": summary,
"rows": [row],
"data_hash": "data",
"target_split_hash": "split",
},
indent=2,
)
+ "\n"
)
def test_ctt_proxy_comparison_writes_full_proxy_gate_without_markdown(
tmp_path, monkeypatch
) -> None:
local = tmp_path / "local_atlas"
ctt = tmp_path / "ctt_residual"
_write_proxy_run(
local,
pptc20=0.4,
pptc40=0.7,
neg20=0.03,
neg40=0.2,
pos_closer=0.6,
mean_pos=0.7,
mean_neg=0.8,
diversity=0.9,
collapse=0.05,
)
_write_proxy_run(
ctt,
pptc20=0.2,
pptc40=0.6,
neg20=0.035,
neg40=0.25,
pos_closer=0.8,
mean_pos=0.4,
mean_neg=0.5,
diversity=0.3,
collapse=0.06,
)
monkeypatch.setattr(
comparison,
"DEFAULT_RUNS",
[("local_atlas", [local]), ("ctt_residual", [ctt])],
)
out_dir = tmp_path / "comparison"
assert comparison.main(["--out-dir", str(out_dir), "--no-markdown-report"]) == 0
metrics = json.loads((out_dir / "metrics.json").read_text())
ctt_row = next(row for row in metrics["rows"] if row["method"] == "ctt_residual")
by_task = json.loads((out_dir / "metrics_by_task.json").read_text())
by_seed = json.loads((out_dir / "metrics_by_seed.json").read_text())
assert ctt_row["proxy_gate_pass"] is True
assert ctt_row["pos_closer_than_neg"] == 0.8
assert ctt_row["mean_negative_distance"] == 0.5
assert ctt_row["collapse_rate"] == 0.06
assert by_task["ctt_residual"]["PickCube-v1"]["pos_closer_than_neg"] == 0.8
assert by_seed["ctt_residual"]["0"]["mean_negative_distance"] == 0.5
assert "Pos<Neg" in (out_dir / "table.tex").read_text()
assert not (out_dir / "report.md").exists()
|