| |
| """Model-card tables from the measured results (D-59), so the card's numbers are copied by code, never by hand. |
| |
| Reads reports/compare/<name>.summary.json (fast suite), .full.summary.json (full suite), .code.summary.json |
| (HumanEval+/MBPP+) and the lineup file (release/lineup.yaml: which result names appear, under which labels, and the |
| phone speeds), and fills the {{TABLE_*}} markers of a card template. |
| |
| python scripts/card_tables.py --template release/card_gguf.md --lineup release/lineup.yaml --out README.md |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import re |
| from pathlib import Path |
|
|
| import yaml |
|
|
| REPO = Path(__file__).resolve().parents[1] |
| CMP = REPO / "reports" / "compare" |
|
|
|
|
| def load(name: str, suffix: str) -> dict | None: |
| p = CMP / f"{name}{suffix}.summary.json" |
| return json.loads(p.read_text()) if p.exists() else None |
|
|
|
|
| def pct(x: float | None) -> str: |
| return "—" if x is None else f"{100 * x:.1f}" |
|
|
|
|
| def gb(n: int | None) -> str: |
| return "—" if not n else f"{n / 1e9:.2f} GB" |
|
|
|
|
| def table(rows: list[dict], cols: list[tuple[str, str, str]], suffix: str, bold: set[str]) -> str: |
| """cols: (header, summary key, group) — key 'groups' entries as percentages.""" |
| head = "| Model | File | " + " | ".join(c[0] for c in cols) + " |" |
| out = [head, "|" + "---|" * (len(cols) + 2)] |
| for r in rows: |
| s = load(r["result"], suffix) |
| if s is None: |
| continue |
| cells = [pct(s["groups"].get(c[1])) for c in cols] |
| label = f"**{r['label']}**" if r["result"] in bold else r["label"] |
| out.append(f"| {label} | {gb(s.get('bytes'))} | " + " | ".join(cells) + " |") |
| return "\n".join(out) |
|
|
|
|
| def main() -> None: |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--template", type=Path, required=True) |
| ap.add_argument("--lineup", type=Path, required=True) |
| ap.add_argument("--out", type=Path, required=True) |
| a = ap.parse_args() |
| L = yaml.safe_load(a.lineup.read_text()) |
| ours = {r["result"] for r in L["ours"]} |
| rows = L["ours"] + L["reference"] + L["others"] |
| tables = { |
| "TABLE_FAST": table(rows, [("MMLU-Redux", "MMLU-Redux", ""), ("Global-MMLU-Lite (5)", "Global-MMLU (5 lang)", ""), |
| ("Belebele (5)", "Belebele (5 lang)", ""), ("GSM8K-250", "GSM8K-250", "")], "", ours), |
| "TABLE_FULL": table(rows, [("MMLU-Pro", "MMLU-Pro (direct)", ""), ("Global-MMLU (5 more)", "Global-MMLU (de es fr ja ru)", ""), |
| ("MATH-500", "MATH-500", ""), ("IFEval (prompt)", "IFEval (prompt strict)", ""), |
| ("IFEval (instr.)", "IFEval (instruction strict)", "")], ".full", ours), |
| "TABLE_CODE": table(rows, [("HumanEval+", "HumanEval+", ""), ("MBPP+", "MBPP+", "")], ".code", ours), |
| } |
| sp = ["| File | Size | Prompt (tok/s) | Generation (tok/s) |", "|---|---|---|---|"] |
| for r in rows: |
| if r.get("pixel"): |
| s = load(r["result"], "") or load(r["result"], ".full") or {} |
| sp.append(f"| {('**' + r['label'] + '**') if r['result'] in ours else r['label']} | {gb(s.get('bytes'))} | " |
| f"{r['pixel'][0]} | {r['pixel'][1]} |") |
| tables["TABLE_PIXEL"] = "\n".join(sp) |
| text = a.template.read_text() |
| for k, v in tables.items(): |
| text = text.replace("{{" + k + "}}", v) |
| left = re.findall(r"\{\{[A-Z_]+\}\}", text) |
| if left: |
| raise SystemExit(f"unfilled markers: {left}") |
| a.out.write_text(text) |
| print(f"wrote {a.out}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
