File size: 3,604 Bytes
e5235fb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | #!/usr/bin/env python3
"""Model-card tables from the measured results (D-59), so the card's numbers are copied by code, never by hand.
Reads reports/compare/<name>.summary.json (fast suite), .full.summary.json (full suite), .code.summary.json
(HumanEval+/MBPP+) and the lineup file (release/lineup.yaml: which result names appear, under which labels, and the
phone speeds), and fills the {{TABLE_*}} markers of a card template.
python scripts/card_tables.py --template release/card_gguf.md --lineup release/lineup.yaml --out README.md
"""
from __future__ import annotations
import argparse
import json
import re
from pathlib import Path
import yaml
REPO = Path(__file__).resolve().parents[1]
CMP = REPO / "reports" / "compare"
def load(name: str, suffix: str) -> dict | None:
p = CMP / f"{name}{suffix}.summary.json"
return json.loads(p.read_text()) if p.exists() else None
def pct(x: float | None) -> str:
return "—" if x is None else f"{100 * x:.1f}"
def gb(n: int | None) -> str:
return "—" if not n else f"{n / 1e9:.2f} GB"
def table(rows: list[dict], cols: list[tuple[str, str, str]], suffix: str, bold: set[str]) -> str:
"""cols: (header, summary key, group) — key 'groups' entries as percentages."""
head = "| Model | File | " + " | ".join(c[0] for c in cols) + " |"
out = [head, "|" + "---|" * (len(cols) + 2)]
for r in rows:
s = load(r["result"], suffix)
if s is None:
continue
cells = [pct(s["groups"].get(c[1])) for c in cols]
label = f"**{r['label']}**" if r["result"] in bold else r["label"]
out.append(f"| {label} | {gb(s.get('bytes'))} | " + " | ".join(cells) + " |")
return "\n".join(out)
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--template", type=Path, required=True)
ap.add_argument("--lineup", type=Path, required=True)
ap.add_argument("--out", type=Path, required=True)
a = ap.parse_args()
L = yaml.safe_load(a.lineup.read_text())
ours = {r["result"] for r in L["ours"]}
rows = L["ours"] + L["reference"] + L["others"]
tables = {
"TABLE_FAST": table(rows, [("MMLU-Redux", "MMLU-Redux", ""), ("Global-MMLU-Lite (5)", "Global-MMLU (5 lang)", ""),
("Belebele (5)", "Belebele (5 lang)", ""), ("GSM8K-250", "GSM8K-250", "")], "", ours),
"TABLE_FULL": table(rows, [("MMLU-Pro", "MMLU-Pro (direct)", ""), ("Global-MMLU (5 more)", "Global-MMLU (de es fr ja ru)", ""),
("MATH-500", "MATH-500", ""), ("IFEval (prompt)", "IFEval (prompt strict)", ""),
("IFEval (instr.)", "IFEval (instruction strict)", "")], ".full", ours),
"TABLE_CODE": table(rows, [("HumanEval+", "HumanEval+", ""), ("MBPP+", "MBPP+", "")], ".code", ours),
}
sp = ["| File | Size | Prompt (tok/s) | Generation (tok/s) |", "|---|---|---|---|"]
for r in rows:
if r.get("pixel"):
s = load(r["result"], "") or load(r["result"], ".full") or {}
sp.append(f"| {('**' + r['label'] + '**') if r['result'] in ours else r['label']} | {gb(s.get('bytes'))} | "
f"{r['pixel'][0]} | {r['pixel'][1]} |")
tables["TABLE_PIXEL"] = "\n".join(sp)
text = a.template.read_text()
for k, v in tables.items():
text = text.replace("{{" + k + "}}", v)
left = re.findall(r"\{\{[A-Z_]+\}\}", text)
if left:
raise SystemExit(f"unfilled markers: {left}")
a.out.write_text(text)
print(f"wrote {a.out}")
if __name__ == "__main__":
main()
|