File size: 3,251 Bytes
0052e99 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | """Download effort-eval results and print accuracy 路 mean tokens per benchmark, next to base Dory and the
earlier blends on the same problems.
MODAL_PROFILE=aside python3 scripts/summarize_effort.py smoke
MODAL_PROFILE=aside python3 scripts/summarize_effort.py full MODEL LOOPS EFFORTS SHARDS
"""
import json
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
RES = ROOT / "data" / "eval" / "results"
BENCHES = ["gpqa", "math500", "aime24", "aime25", "amc23", "heldout"]
def fetch(model, name):
with tempfile.TemporaryDirectory() as tmp:
p = subprocess.run(["modal", "volume", "get", "beam-dory-data",
f"eval/results/{model.replace('/', '_')}/{name}", tmp], capture_output=True)
f = Path(tmp) / name
return [json.loads(l) for l in open(f)] if p.returncode == 0 and f.exists() else None
def row(label, rs, ids=None):
if ids is not None:
rs = [r for r in rs if r["id"] in ids]
cells = []
for b in BENCHES:
x = [r for r in rs if r["bench"] == b]
cells.append(f"{b} {100 * sum(r['correct'] for r in x) / len(x):5.1f}路{sum(r['tokens'] for r in x) / len(x) / 1e3:4.1f}k"
if x else f"{b} -")
tok = sum(r["tokens"] for r in rs) / len(rs)
acc = 100 * sum(r["correct"] for r in rs) / len(rs)
print(f"{label:24s} " + " ".join(cells) + f" | all {acc:5.1f}路{tok / 1e3:4.2f}k cut {sum(r['finish'] == 'length' for r in rs)}")
return rs
def main():
mode = sys.argv[1]
if mode == "smoke":
runs = [(m, 2, e, f"loops2{'' if e == 'default' else '-' + e}-limit.jsonl")
for m in ("merges/rE-t050", "merges/rE-t030") for e in ("low", "default", "high")]
else:
_, _, model, loops, efforts, shards = sys.argv
runs = [(model, int(loops), e, [f"loops{loops}{'' if e == 'default' else '-' + e}-s{i}of{shards}.jsonl"
for i in range(int(shards))]) for e in efforts.split(",")]
results = {}
for model, loops, effort, names in runs:
names = [names] if isinstance(names, str) else names
parts = [fetch(model, n) for n in names]
if all(parts):
results[(model, loops, effort)] = [r for p in parts for r in p]
if mode == "full":
out = RES / f"{model.split('/')[-1]}_loops{loops}-{effort}-64k.jsonl"
out.write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in results[(model, loops, effort)]))
else:
print(f"missing: {model} {effort}")
ids = {r["id"] for rs in results.values() for r in rs}
loops_set = sorted({k[1] for k in results})
for loops in loops_set:
print(f"\n{loops} loops, accuracy 路 mean output tokens" + (" (same problems)" if mode == "smoke" else ""))
for ref in ("base", "mix030", "mix050"):
f = RES / f"{ref}_loops{loops}-64k.jsonl"
if f.exists():
row(ref, [json.loads(l) for l in open(f)], ids)
for (model, l, effort), rs in results.items():
if l == loops:
row(f"{model.split('/')[-1]} {effort}", rs)
if __name__ == "__main__":
main()
|