| """Cross-run comparison table (markdown) from files each run writes.""" |
| from __future__ import annotations |
|
|
| import json |
| from pathlib import Path |
|
|
|
|
| def _load(p: Path): |
| return json.loads(p.read_text()) if p.exists() else None |
|
|
|
|
| def run_row(run: str | Path) -> dict: |
| run = Path(run) |
| s = _load(run / "summary.json") or {} |
| g = _load(run / "games.json") or {} |
| b = _load(run / "benchmark.json") or [] |
| cal = s.get("test_calibrated", {}) |
| row = {"run": run.name, "model": (_load(run / "env.json") or {}).get("task", ""), |
| "params": s.get("params_human"), "acc": cal.get("accuracy"), "ece": cal.get("ece"), |
| "ece_raw": s.get("test_raw", {}).get("ece"), "nll": cal.get("nll"), "brier": cal.get("brier"), |
| "avg_score": g.get("avg_score"), "median": g.get("median_score"), "max": g.get("max_score"), |
| "win": g.get("win_rate"), "fatal": g.get("fatal_move_rate"), |
| "train_min": round(s["train_seconds"] / 60, 1) if s.get("train_seconds") else None, |
| "cost": (s.get("cost") or {}).get("actual_cost", (s.get("cost") or {}).get("estimated_cost"))} |
| for r in b: |
| kind = "int8" if ".int8." in r["model"] else ("onnx" if r["model"].endswith(".onnx") else "torch") |
| row[f"{kind}_mb"] = r["file_size_mb"] |
| row[f"{kind}_p50"] = r["latency"]["end_to_end"]["p50_ms"] |
| row[f"{kind}_p99"] = r["latency"]["end_to_end"]["p99_ms"] |
| row[f"{kind}_dps"] = r["latency"]["max_decisions_per_sec_batch1"] |
| return row |
|
|
|
|
| COLUMNS = [("run", "run"), ("params", "params"), ("onnx_mb", "onnx MB"), ("int8_mb", "int8 MB"), |
| ("onnx_p50", "p50 ms"), ("onnx_p99", "p99 ms"), ("onnx_dps", "dec/s"), ("acc", "acc"), |
| ("ece_raw", "ECE raw"), ("ece", "ECE"), ("brier", "Brier"), ("avg_score", "avg score"), |
| ("median", "median"), ("max", "max"), ("win", "win"), ("fatal", "fatal-move"), |
| ("train_min", "train min"), ("cost", "cost €")] |
|
|
|
|
| def _fmt(v, key): |
| if v is None: |
| return "-" |
| if key in ("acc", "win"): |
| return f"{v:.1%}" |
| if key == "fatal": |
| return f"{v:.3%}" |
| if key in ("ece", "ece_raw", "brier", "nll"): |
| return f"{v:.4f}" |
| if isinstance(v, float): |
| return f"{v:.2f}" if v < 100 else f"{v:,.0f}" |
| return str(v) |
|
|
|
|
| def compare_runs(runs, out: str | None = None) -> str: |
| rows = [run_row(r) for r in runs] |
| cols = [(k, h) for k, h in COLUMNS if any(r.get(k) is not None for r in rows)] |
| lines = ["| " + " | ".join(h for _, h in cols) + " |", "|" + "---|" * len(cols)] |
| for r in rows: |
| lines.append("| " + " | ".join(_fmt(r.get(k), k) for k, _ in cols) + " |") |
| table = "\n".join(lines) |
| if out: |
| Path(out).write_text(table + "\n") |
| return table |
|
|
