Re-evaluate at a 64k-token limit: resume answers cut off at 32k (corrects AIME comparisons)
d33ce47 verified | """Resume every answer that hit the 32k limit up to 64k tokens, for base Dory and Dory-Beam-4B at 1, 2 and 4 loops | |
| plus IFEval, one container per config. Reads the truncated records from data/eval/results/. | |
| MODAL_PROFILE=aside modal deploy modal_app/eval_dory.py::APP | |
| MODAL_PROFILE=aside python3 scripts/spawn_continue.py [--smoke] | |
| Results land on the beam-dory-data Volume as eval/results/<model>/[ifeval_]loops<L>-cont64k.jsonl. | |
| """ | |
| import json | |
| import sys | |
| from pathlib import Path | |
| import modal | |
| RES = Path(__file__).resolve().parent.parent / "data" / "eval" / "results" | |
| MODELS = {"base": "base", "mix050": "merges/rA-t050"} | |
| CONFIGS = [(tag, l, "bench") for tag in MODELS for l in (1, 2, 4)] + [(tag, 2, "ifeval") for tag in MODELS] | |
| smoke = "--smoke" in sys.argv | |
| fn = modal.Function.from_name("beam-dory-eval", "continue_truncated", environment_name="main") | |
| for tag, loops, kind in ([("base", 2, "bench")] if smoke else CONFIGS): | |
| f = RES / f"{tag}_{'ifeval_' if kind == 'ifeval' else ''}loops{loops}.jsonl" | |
| recs = [r for r in map(json.loads, open(f)) if r["finish"] == "length"] | |
| if smoke: # 3 records, 256 more tokens each: checks the resumed text continues coherently | |
| recs = recs[:3] | |
| call = fn.spawn(MODELS[tag], loops, recs, kind, (recs[0]["tokens"] + 256 + 1023) // 1024 * 1024) | |
| else: | |
| call = fn.spawn(MODELS[tag], loops, recs, kind, 65536) | |
| print(json.dumps({"tag": tag, "loops": loops, "kind": kind, "n": len(recs), "call": call.object_id})) | |