| from __future__ import annotations |
|
|
| import json |
| from pathlib import Path |
|
|
| from scripts import build_ctt_proxy_comparison as comparison |
|
|
|
|
| def _write_proxy_run( |
| run_dir: Path, |
| *, |
| pptc20: float, |
| pptc40: float, |
| neg20: float, |
| neg40: float, |
| pos_closer: float, |
| mean_pos: float, |
| mean_neg: float, |
| diversity: float, |
| collapse: float, |
| ) -> None: |
| run_dir.mkdir(parents=True) |
| row = { |
| "chart_id": "c0", |
| "task_id": "PickCube-v1", |
| "seed": "0", |
| "pptc_at_16_thr_0p20": pptc20, |
| "pptc_at_16_thr_0p40": pptc40, |
| "negative_near_at_16_thr_0p20": neg20, |
| "negative_near_at_16_thr_0p40": neg40, |
| "pos_closer_than_neg_at_16": pos_closer, |
| "mean_positive_distance_at_16": mean_pos, |
| "mean_negative_distance_at_16": mean_neg, |
| "candidate_diversity_at_16": diversity, |
| "collapse_rate_at_16": collapse, |
| "proxy_support_distance_at_16": mean_pos, |
| } |
| summary = { |
| key: {"micro": {"mean": value}} |
| for key, value in row.items() |
| if isinstance(value, float) |
| } |
| (run_dir / "metrics.json").write_text( |
| json.dumps( |
| { |
| "num_rows": 1, |
| "summary": summary, |
| "rows": [row], |
| "data_hash": "data", |
| "target_split_hash": "split", |
| }, |
| indent=2, |
| ) |
| + "\n" |
| ) |
|
|
|
|
| def test_ctt_proxy_comparison_writes_full_proxy_gate_without_markdown( |
| tmp_path, monkeypatch |
| ) -> None: |
| local = tmp_path / "local_atlas" |
| ctt = tmp_path / "ctt_residual" |
| _write_proxy_run( |
| local, |
| pptc20=0.4, |
| pptc40=0.7, |
| neg20=0.03, |
| neg40=0.2, |
| pos_closer=0.6, |
| mean_pos=0.7, |
| mean_neg=0.8, |
| diversity=0.9, |
| collapse=0.05, |
| ) |
| _write_proxy_run( |
| ctt, |
| pptc20=0.2, |
| pptc40=0.6, |
| neg20=0.035, |
| neg40=0.25, |
| pos_closer=0.8, |
| mean_pos=0.4, |
| mean_neg=0.5, |
| diversity=0.3, |
| collapse=0.06, |
| ) |
| monkeypatch.setattr( |
| comparison, |
| "DEFAULT_RUNS", |
| [("local_atlas", [local]), ("ctt_residual", [ctt])], |
| ) |
|
|
| out_dir = tmp_path / "comparison" |
| assert comparison.main(["--out-dir", str(out_dir), "--no-markdown-report"]) == 0 |
|
|
| metrics = json.loads((out_dir / "metrics.json").read_text()) |
| ctt_row = next(row for row in metrics["rows"] if row["method"] == "ctt_residual") |
| by_task = json.loads((out_dir / "metrics_by_task.json").read_text()) |
| by_seed = json.loads((out_dir / "metrics_by_seed.json").read_text()) |
|
|
| assert ctt_row["proxy_gate_pass"] is True |
| assert ctt_row["pos_closer_than_neg"] == 0.8 |
| assert ctt_row["mean_negative_distance"] == 0.5 |
| assert ctt_row["collapse_rate"] == 0.06 |
| assert by_task["ctt_residual"]["PickCube-v1"]["pos_closer_than_neg"] == 0.8 |
| assert by_seed["ctt_residual"]["0"]["mean_negative_distance"] == 0.5 |
| assert "Pos<Neg" in (out_dir / "table.tex").read_text() |
| assert not (out_dir / "report.md").exists() |
|
|
