File size: 2,759 Bytes
026f63e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | """Cross-run comparison table (markdown) from files each run writes."""
from __future__ import annotations
import json
from pathlib import Path
def _load(p: Path):
return json.loads(p.read_text()) if p.exists() else None
def run_row(run: str | Path) -> dict:
run = Path(run)
s = _load(run / "summary.json") or {}
g = _load(run / "games.json") or {}
b = _load(run / "benchmark.json") or []
cal = s.get("test_calibrated", {})
row = {"run": run.name, "model": (_load(run / "env.json") or {}).get("task", ""),
"params": s.get("params_human"), "acc": cal.get("accuracy"), "ece": cal.get("ece"),
"ece_raw": s.get("test_raw", {}).get("ece"), "nll": cal.get("nll"), "brier": cal.get("brier"),
"avg_score": g.get("avg_score"), "median": g.get("median_score"), "max": g.get("max_score"),
"win": g.get("win_rate"), "fatal": g.get("fatal_move_rate"),
"train_min": round(s["train_seconds"] / 60, 1) if s.get("train_seconds") else None,
"cost": (s.get("cost") or {}).get("actual_cost", (s.get("cost") or {}).get("estimated_cost"))}
for r in b:
kind = "int8" if ".int8." in r["model"] else ("onnx" if r["model"].endswith(".onnx") else "torch")
row[f"{kind}_mb"] = r["file_size_mb"]
row[f"{kind}_p50"] = r["latency"]["end_to_end"]["p50_ms"]
row[f"{kind}_p99"] = r["latency"]["end_to_end"]["p99_ms"]
row[f"{kind}_dps"] = r["latency"]["max_decisions_per_sec_batch1"]
return row
COLUMNS = [("run", "run"), ("params", "params"), ("onnx_mb", "onnx MB"), ("int8_mb", "int8 MB"),
("onnx_p50", "p50 ms"), ("onnx_p99", "p99 ms"), ("onnx_dps", "dec/s"), ("acc", "acc"),
("ece_raw", "ECE raw"), ("ece", "ECE"), ("brier", "Brier"), ("avg_score", "avg score"),
("median", "median"), ("max", "max"), ("win", "win"), ("fatal", "fatal-move"),
("train_min", "train min"), ("cost", "cost €")]
def _fmt(v, key):
if v is None:
return "-"
if key in ("acc", "win"):
return f"{v:.1%}"
if key == "fatal":
return f"{v:.3%}"
if key in ("ece", "ece_raw", "brier", "nll"):
return f"{v:.4f}"
if isinstance(v, float):
return f"{v:.2f}" if v < 100 else f"{v:,.0f}"
return str(v)
def compare_runs(runs, out: str | None = None) -> str:
rows = [run_row(r) for r in runs]
cols = [(k, h) for k, h in COLUMNS if any(r.get(k) is not None for r in rows)]
lines = ["| " + " | ".join(h for _, h in cols) + " |", "|" + "---|" * len(cols)]
for r in rows:
lines.append("| " + " | ".join(_fmt(r.get(k), k) for k, _ in cols) + " |")
table = "\n".join(lines)
if out:
Path(out).write_text(table + "\n")
return table
|