File size: 4,835 Bytes
d33ce47 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | """Merge the 64k continuations (scripts/spawn_continue.py) into full result files, then print accuracy and tokens
at both limits plus paired bootstrap CIs, Dory-Beam-4B minus base, at 64k.
MODAL_PROFILE=aside python3 scripts/merge_continued.py
Writes data/eval/results/<tag>_[ifeval_]loops<L>-64k.jsonl: the original records, with every answer that hit the
32k limit (16k for IFEval) replaced by its continuation.
"""
import json
import random
import subprocess
import tempfile
from collections import defaultdict
from pathlib import Path
RES = Path(__file__).resolve().parent.parent / "data" / "eval" / "results"
MODELS = {"base": "base", "mix050": "merges_rA-t050"}
BENCHES = ["gpqa", "math500", "aime24", "aime25", "amc23", "heldout"]
def key(r):
return (r["key"], r["sample"]) if "key" in r else (r["bench"], r["id"], r["sample"])
def merge(tag, loops, kind):
pre = "ifeval_" if kind == "ifeval" else ""
with tempfile.TemporaryDirectory() as tmp:
subprocess.run(["modal", "volume", "get", "beam-dory-data",
f"eval/results/{MODELS[tag]}/{pre}loops{loops}-cont64k.jsonl", tmp], check=True,
capture_output=True)
cont = {key(r): r for r in map(json.loads, open(Path(tmp) / f"{pre}loops{loops}-cont64k.jsonl"))}
orig = [json.loads(l) for l in open(RES / f"{tag}_{pre}loops{loops}.jsonl")]
assert set(cont) == {key(r) for r in orig if r["finish"] == "length"}, (tag, loops, kind)
out = [cont.get(key(r), r) for r in orig]
with open(RES / f"{tag}_{pre}loops{loops}-64k.jsonl", "w") as f:
for r in out:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
return orig, out
def row(rs, bench):
rs = [r for r in rs if r["bench"] == bench]
return (100 * sum(r["correct"] for r in rs) / len(rs), sum(r["tokens"] for r in rs) / len(rs),
100 * sum(r["finish"] == "length" for r in rs) / len(rs))
def paired_ci(a, b, bench, n_boot=4000):
pa, pb = defaultdict(list), defaultdict(list)
for r in a:
if r["bench"] == bench:
pa[r["id"]].append(r["correct"])
for r in b:
if r["bench"] == bench:
pb[r["id"]].append(r["correct"])
d = {i: sum(pa[i]) / len(pa[i]) - sum(pb[i]) / len(pb[i]) for i in pa}
ids, rng = sorted(d), random.Random(0)
boots = sorted(sum(d[i] for i in s) / len(s) for s in ([rng.choice(ids) for _ in ids] for _ in range(n_boot)))
return 100 * sum(d.values()) / len(d), 100 * boots[int(0.025 * n_boot)], 100 * boots[int(0.975 * n_boot)]
def main():
runs = {}
for tag in MODELS:
for loops in (1, 2, 4):
runs[(tag, loops)] = merge(tag, loops, "bench")
runs[(tag, "ifeval")] = merge(tag, 2, "ifeval")
print("accuracy 路 mean tokens (% cut off), 32k limit -> 64k limit")
for loops in (1, 2, 4):
for tag in MODELS:
o, n = runs[(tag, loops)]
cells = [f"{b} {row(o, b)[0]:.1f}路{row(o, b)[1] / 1e3:.1f}k({row(o, b)[2]:.0f}%)"
f" -> {row(n, b)[0]:.1f}路{row(n, b)[1] / 1e3:.1f}k({row(n, b)[2]:.1f}%)" for b in BENCHES]
print(f"{tag:7s} L{loops} " + " ".join(cells))
for tag in MODELS:
o, n = runs[(tag, "ifeval")]
s = lambda rs: (100 * sum(r["strict_prompt"] for r in rs) / len(rs), sum(r["tokens"] for r in rs) / len(rs),
sum(r["finish"] == "length" for r in rs))
print(f"{tag:7s} IFEval prompt-strict 16k {s(o)[0]:.1f}路{s(o)[1] / 1e3:.1f}k({s(o)[2]} cut)"
f" -> 64k {s(n)[0]:.1f}路{s(n)[1] / 1e3:.1f}k({s(n)[2]} cut)")
print("\nDory-Beam-4B minus base at 64k, points, 95% paired bootstrap CI over problems (* = excludes 0)")
for loops in (1, 2, 4):
a, b = runs[("mix050", loops)][1], runs[("base", loops)][1]
cells = []
for bench in BENCHES:
d, lo, hi = paired_ci(a, b, bench)
cells.append(f"{bench} {d:+.1f} [{lo:+.1f},{hi:+.1f}]{'*' if not lo < 0 < hi else ''}")
ratio = sum(r["tokens"] for r in b) / sum(r["tokens"] for r in a)
print(f"L{loops} " + " ".join(cells) + f" | base/beam tokens {ratio:.2f}x")
a, b = runs[("mix050", "ifeval")][1], runs[("base", "ifeval")][1]
pa = {r["key"]: [] for r in a}
pb = {r["key"]: [] for r in b}
for r in a:
pa[r["key"]].append(r["strict_prompt"])
for r in b:
pb[r["key"]].append(r["strict_prompt"])
d = {k: sum(pa[k]) / len(pa[k]) - sum(pb[k]) / len(pb[k]) for k in pa}
ids, rng = sorted(d), random.Random(0)
boots = sorted(sum(d[i] for i in s) / len(s) for s in ([rng.choice(ids) for _ in ids] for _ in range(4000)))
print(f"IFEval prompt-strict {100 * sum(d.values()) / len(d):+.1f} [{100 * boots[100]:+.1f},{100 * boots[3900]:+.1f}]")
if __name__ == "__main__":
main()
|