"""Cross-run comparison table (markdown) from files each run writes.""" from __future__ import annotations import json from pathlib import Path def _load(p: Path): return json.loads(p.read_text()) if p.exists() else None def run_row(run: str | Path) -> dict: run = Path(run) s = _load(run / "summary.json") or {} g = _load(run / "games.json") or {} b = _load(run / "benchmark.json") or [] cal = s.get("test_calibrated", {}) row = {"run": run.name, "model": (_load(run / "env.json") or {}).get("task", ""), "params": s.get("params_human"), "acc": cal.get("accuracy"), "ece": cal.get("ece"), "ece_raw": s.get("test_raw", {}).get("ece"), "nll": cal.get("nll"), "brier": cal.get("brier"), "avg_score": g.get("avg_score"), "median": g.get("median_score"), "max": g.get("max_score"), "win": g.get("win_rate"), "fatal": g.get("fatal_move_rate"), "train_min": round(s["train_seconds"] / 60, 1) if s.get("train_seconds") else None, "cost": (s.get("cost") or {}).get("actual_cost", (s.get("cost") or {}).get("estimated_cost"))} for r in b: kind = "int8" if ".int8." in r["model"] else ("onnx" if r["model"].endswith(".onnx") else "torch") row[f"{kind}_mb"] = r["file_size_mb"] row[f"{kind}_p50"] = r["latency"]["end_to_end"]["p50_ms"] row[f"{kind}_p99"] = r["latency"]["end_to_end"]["p99_ms"] row[f"{kind}_dps"] = r["latency"]["max_decisions_per_sec_batch1"] return row COLUMNS = [("run", "run"), ("params", "params"), ("onnx_mb", "onnx MB"), ("int8_mb", "int8 MB"), ("onnx_p50", "p50 ms"), ("onnx_p99", "p99 ms"), ("onnx_dps", "dec/s"), ("acc", "acc"), ("ece_raw", "ECE raw"), ("ece", "ECE"), ("brier", "Brier"), ("avg_score", "avg score"), ("median", "median"), ("max", "max"), ("win", "win"), ("fatal", "fatal-move"), ("train_min", "train min"), ("cost", "cost €")] def _fmt(v, key): if v is None: return "-" if key in ("acc", "win"): return f"{v:.1%}" if key == "fatal": return f"{v:.3%}" if key in ("ece", "ece_raw", "brier", "nll"): return f"{v:.4f}" if isinstance(v, float): return f"{v:.2f}" if v < 100 else f"{v:,.0f}" return str(v) def compare_runs(runs, out: str | None = None) -> str: rows = [run_row(r) for r in runs] cols = [(k, h) for k, h in COLUMNS if any(r.get(k) is not None for r in rows)] lines = ["| " + " | ".join(h for _, h in cols) + " |", "|" + "---|" * len(cols)] for r in rows: lines.append("| " + " | ".join(_fmt(r.get(k), k) for k, _ in cols) + " |") table = "\n".join(lines) if out: Path(out).write_text(table + "\n") return table