code/scripts/spawn_continue.py

"""Resume every answer that hit the 32k limit up to 64k tokens, for base Dory and Dory-Beam-4B at 1, 2 and 4 loops
plus IFEval, one container per config. Reads the truncated records from data/eval/results/.

  MODAL_PROFILE=aside modal deploy modal_app/eval_dory.py::APP
  MODAL_PROFILE=aside python3 scripts/spawn_continue.py [--smoke]

Results land on the beam-dory-data Volume as eval/results/<model>/[ifeval_]loops<L>-cont64k.jsonl.
"""
import json
import sys
from pathlib import Path

import modal

RES = Path(__file__).resolve().parent.parent / "data" / "eval" / "results"
MODELS = {"base": "base", "mix050": "merges/rA-t050"}
CONFIGS = [(tag, l, "bench") for tag in MODELS for l in (1, 2, 4)] + [(tag, 2, "ifeval") for tag in MODELS]

smoke = "--smoke" in sys.argv
fn = modal.Function.from_name("beam-dory-eval", "continue_truncated", environment_name="main")
for tag, loops, kind in ([("base", 2, "bench")] if smoke else CONFIGS):
    f = RES / f"{tag}_{'ifeval_' if kind == 'ifeval' else ''}loops{loops}.jsonl"
    recs = [r for r in map(json.loads, open(f)) if r["finish"] == "length"]
    if smoke:  # 3 records, 256 more tokens each: checks the resumed text continues coherently
        recs = recs[:3]
        call = fn.spawn(MODELS[tag], loops, recs, kind, (recs[0]["tokens"] + 256 + 1023) // 1024 * 1024)
    else:
        call = fn.spawn(MODELS[tag], loops, recs, kind, 65536)
    print(json.dumps({"tag": tag, "loops": loops, "kind": kind, "n": len(recs), "call": call.object_id}))