| """Evaluate Dory checkpoints with vLLM (Dory support from vllm-project/vllm#60149) on Modal. |
| |
| modal run modal_app/eval_dory.py --models base --loops 2 --limit 20 # smoke |
| modal run modal_app/eval_dory.py --models base,runs/r1/final --loops 1,2,4 |
| |
| One H100 container per (model, loops) config, run in parallel. Each sampled answer is |
| scored with scripts/verify.py; per-sample records go to the beam-dory-data Volume under |
| eval/results/<model>/loops<L>.jsonl. Reports accuracy and output tokens per benchmark. |
| """ |
|
|
| import json |
| import os |
| import shutil |
| import sys |
| import time |
| from collections import defaultdict |
| from pathlib import Path |
|
|
| import modal |
|
|
| APP = modal.App("beam-dory-eval") |
| weights = modal.Volume.from_name("beam-weights", create_if_missing=True, version=2) |
| data = modal.Volume.from_name("beam-dory-data", create_if_missing=True, version=2) |
| ckpt = modal.Volume.from_name("beam-dory-ckpt", create_if_missing=True, version=2) |
| W, D, C = "/weights", "/data", "/ckpt" |
| VLLM_PR_SHA = "ce9dd0ed884b7cb4014396c1aa715a8b7c9fc4a4" |
|
|
| image = ( |
| |
| modal.Image.from_registry("nvidia/cuda:13.0.3-devel-ubuntu24.04", add_python="3.12") |
| .apt_install("git") |
| .env({"VLLM_USE_PRECOMPILED": "1"}) |
| .pip_install(f"vllm @ git+https://github.com/vllm-project/vllm.git@{VLLM_PR_SHA}", |
| "math-verify[antlr4_13_2]") |
| .pip_install("langdetect", "immutabledict", "nltk>=3.9.1") |
| .env({"PYTHONUNBUFFERED": "1", "TOKENIZERS_PARALLELISM": "false", |
| "FLASHINFER_WORKSPACE_BASE": "/data/cache/flashinfer"}) |
| .add_local_file("scripts/verify.py", "/proj/verify.py") |
| .add_local_dir("third_party/ifeval", "/proj/ifeval") |
| ) |
|
|
| TOKENIZER_FILES = ("tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "chat_template.jinja") |
|
|
|
|
| def model_dir(name): |
| return f"{W}/dory-4b" if name == "base" else f"{C}/{name}" |
|
|
|
|
| def load_llm(model, loops, max_tokens): |
| from vllm import LLM |
|
|
| path = model_dir(model) |
| |
| tok_dir = "/tmp/dory-tok" |
| os.makedirs(tok_dir, exist_ok=True) |
| for f in TOKENIZER_FILES: |
| if os.path.exists(f"{path}/{f}"): |
| shutil.copy(f"{path}/{f}", tok_dir) |
| return LLM(model=path, tokenizer=tok_dir, max_model_len=max_tokens + 4096, gpu_memory_utilization=0.92, |
| max_num_seqs=256, enable_prefix_caching=True, |
| hf_overrides={"n_recurrent_loops": loops, "recurrent_kv_cache_mode": "per_loop"}) |
|
|
|
|
| @APP.function(image=image, gpu="H100", volumes={W: weights, D: data, C: ckpt}, timeout=4 * 3600, |
| cpu=8, memory=65536) |
| def evaluate(model: str, loops: int, limit: int = 0, max_tokens: int = 32768, medium_effort: bool = False, |
| benches: str = "", shard: int = 0, num_shards: int = 1, reasoning_effort: str = ""): |
| """num_shards > 1 evaluates every num_shards-th problem (interleaved, so each shard mixes benchmarks). |
| reasoning_effort (low/medium/high) is passed to Dory-Beam's chat template, which tags the user turn.""" |
| sys.path.insert(0, "/proj") |
| from verify import check |
| from vllm import SamplingParams |
|
|
| rows = [json.loads(l) for l in open(f"{D}/eval/evalsets.jsonl")] |
| if benches: |
| rows = [r for r in rows if r["bench"] in benches.split(",")] |
| if limit: |
| per, kept = defaultdict(int), [] |
| for r in rows: |
| per[r["bench"]] += 1 |
| if per[r["bench"]] <= limit: |
| kept.append(r) |
| rows = kept |
| if num_shards > 1: |
| rows = rows[shard::num_shards] |
|
|
| t0 = time.time() |
| llm = load_llm(model, loops, max_tokens) |
| load_s = time.time() - t0 |
|
|
| by_k = defaultdict(list) |
| for r in rows: |
| by_k[r["samples"]].append(r) |
| records = [] |
| t1 = time.time() |
| kwargs = {"enable_thinking": True, "medium_effort": medium_effort} |
| if reasoning_effort: |
| kwargs["reasoning_effort"] = reasoning_effort |
| for k, group in sorted(by_k.items()): |
| sp = SamplingParams(n=k, temperature=1.0, top_p=0.95, max_tokens=max_tokens) |
| outs = llm.chat([[{"role": "user", "content": r["prompt"]}] for r in group], sp, |
| chat_template_kwargs=kwargs, use_tqdm=False) |
| for r, out in zip(group, outs): |
| for j, o in enumerate(out.outputs): |
| text = o.text |
| answer = text.split("</think>")[-1] if "</think>" in text else "" |
| ok, pred = check(answer, r["answer"], r["answer_type"]) |
| records.append({"bench": r["bench"], "id": r["id"], "sample": j, "correct": bool(ok), |
| "pred": pred, "tokens": len(o.token_ids), "finish": o.finish_reason, |
| "thinking_closed": "</think>" in text, "text": text}) |
| gen_s = time.time() - t1 |
|
|
| out_dir = Path(D) / "eval" / "results" / model.replace("/", "_") |
| out_dir.mkdir(parents=True, exist_ok=True) |
| tag = ("eff" if medium_effort else "") + (f"-{reasoning_effort}" if reasoning_effort else "") |
| |
| part = (f"-{benches.replace(',', '+')}" if benches else "") + (f"-s{shard}of{num_shards}" if num_shards > 1 else "") |
| with open(out_dir / f"loops{loops}{tag}{'-limit' if limit else ''}{part}.jsonl", "w") as f: |
| for rec in records: |
| f.write(json.dumps(rec, ensure_ascii=False) + "\n") |
| data.commit() |
|
|
| summary = {"model": model, "loops": loops, "medium_effort": medium_effort, |
| "reasoning_effort": reasoning_effort, "load_s": round(load_s), |
| "gen_s": round(gen_s), "tok_per_s": round(sum(r["tokens"] for r in records) / max(gen_s, 1)), |
| "benches": {}} |
| for b in sorted({r["bench"] for r in records}): |
| rs = [r for r in records if r["bench"] == b] |
| summary["benches"][b] = {"n": len(rs), "acc": round(sum(r["correct"] for r in rs) / len(rs), 4), |
| "mean_tokens": round(sum(r["tokens"] for r in rs) / len(rs)), |
| "truncated": round(sum(r["finish"] == "length" for r in rs) / len(rs), 3)} |
| print("SUMMARY", json.dumps(summary), flush=True) |
| return summary |
|
|
|
|
| @APP.function(image=image, gpu="H100", volumes={W: weights, D: data, C: ckpt}, timeout=4 * 3600, |
| cpu=8, memory=65536) |
| def evaluate_ifeval(model: str, loops: int, samples: int = 2, max_tokens: int = 16384): |
| """IFEval (google/IFEval, 541 prompts) with lm-evaluation-harness's checkers, scoring the text after |
| </think>. Strict and loose, prompt- and instruction-level; a response with no closed think block fails.""" |
| sys.path.insert(0, "/proj") |
| from huggingface_hub import hf_hub_download |
| from ifeval.utils import InputExample, test_instruction_following_loose, test_instruction_following_strict |
| from vllm import SamplingParams |
|
|
| rows = [json.loads(l) for l in open(hf_hub_download("google/IFEval", "ifeval_input_data.jsonl", |
| repo_type="dataset"))] |
| llm = load_llm(model, loops, max_tokens) |
| t1 = time.time() |
| sp = SamplingParams(n=samples, temperature=1.0, top_p=0.95, max_tokens=max_tokens) |
| outs = llm.chat([[{"role": "user", "content": r["prompt"]}] for r in rows], sp, |
| chat_template_kwargs={"enable_thinking": True}, use_tqdm=False) |
| records = [] |
| for r, out in zip(rows, outs): |
| inp = InputExample(key=r["key"], instruction_id_list=r["instruction_id_list"], prompt=r["prompt"], |
| kwargs=r["kwargs"]) |
| for j, o in enumerate(out.outputs): |
| answer = o.text.split("</think>")[-1].strip() if "</think>" in o.text else "" |
| strict, loose = test_instruction_following_strict(inp, answer), test_instruction_following_loose(inp, answer) |
| records.append({"key": r["key"], "sample": j, "tokens": len(o.token_ids), "finish": o.finish_reason, |
| "strict_prompt": strict.follow_all_instructions, |
| "strict_inst": strict.follow_instruction_list, |
| "loose_prompt": loose.follow_all_instructions, |
| "loose_inst": loose.follow_instruction_list, "text": o.text}) |
| gen_s = time.time() - t1 |
|
|
| out_dir = Path(D) / "eval" / "results" / model.replace("/", "_") |
| out_dir.mkdir(parents=True, exist_ok=True) |
| with open(out_dir / f"ifeval_loops{loops}.jsonl", "w") as f: |
| for rec in records: |
| f.write(json.dumps(rec, ensure_ascii=False) + "\n") |
| data.commit() |
|
|
| inst = lambda k: [x for rec in records for x in rec[k]] |
| summary = {"model": model, "loops": loops, "gen_s": round(gen_s), "n": len(records), |
| "prompt_strict": round(sum(r["strict_prompt"] for r in records) / len(records), 4), |
| "inst_strict": round(sum(inst("strict_inst")) / len(inst("strict_inst")), 4), |
| "prompt_loose": round(sum(r["loose_prompt"] for r in records) / len(records), 4), |
| "inst_loose": round(sum(inst("loose_inst")) / len(inst("loose_inst")), 4), |
| "mean_tokens": round(sum(r["tokens"] for r in records) / len(records)), |
| "truncated": round(sum(r["finish"] == "length" for r in records) / len(records), 3)} |
| print("SUMMARY", json.dumps(summary), flush=True) |
| return summary |
|
|
|
|
| @APP.function(image=image, gpu="H100", volumes={W: weights, D: data, C: ckpt}, timeout=4 * 3600, |
| cpu=8, memory=65536) |
| def continue_truncated(model: str, loops: int, records: list, kind: str = "bench", max_total: int = 65536): |
| """Resume answers that hit the original max_tokens from where they stopped, up to max_total output tokens. |
| Same sampling settings, so each result is distributed as if the run had used max_total from the start. |
| `records` are the truncated per-sample records from evaluate (kind="bench") or evaluate_ifeval |
| (kind="ifeval"); returns and saves them with the continued text, re-scored.""" |
| sys.path.insert(0, "/proj") |
| from vllm import SamplingParams, TokensPrompt |
|
|
| if kind == "bench": |
| from verify import check |
| gold = {(r["bench"], r["id"]): r for r in map(json.loads, open(f"{D}/eval/evalsets.jsonl"))} |
| prompt_of = lambda rec: gold[(rec["bench"], rec["id"])]["prompt"] |
| else: |
| from huggingface_hub import hf_hub_download |
| from ifeval.utils import InputExample, test_instruction_following_loose, test_instruction_following_strict |
| gold = {r["key"]: r for r in map(json.loads, open(hf_hub_download( |
| "google/IFEval", "ifeval_input_data.jsonl", repo_type="dataset")))} |
| prompt_of = lambda rec: gold[rec["key"]]["prompt"] |
|
|
| llm = load_llm(model, loops, max_total) |
| tok = llm.get_tokenizer() |
| prompts, params, retok = [], [], [] |
| for rec in records: |
| prefix = tok.apply_chat_template([{"role": "user", "content": prompt_of(rec)}], tokenize=False, |
| add_generation_prompt=True, enable_thinking=True, medium_effort=False) |
| gen_ids = tok.encode(rec["text"], add_special_tokens=False) |
| retok.append(len(gen_ids) - rec["tokens"]) |
| ids = tok.encode(prefix, add_special_tokens=False) + gen_ids |
| prompts.append(TokensPrompt(prompt_token_ids=ids)) |
| params.append(SamplingParams(n=1, temperature=1.0, top_p=0.95, max_tokens=max_total - rec["tokens"])) |
| t1 = time.time() |
| outs = llm.generate(prompts, params, use_tqdm=False) |
| gen_s = time.time() - t1 |
|
|
| out = [] |
| for rec, o in zip(records, outs): |
| o = o.outputs[0] |
| text = rec["text"] + o.text |
| new = {**rec, "tokens": rec["tokens"] + len(o.token_ids), "finish": o.finish_reason, "text": text, |
| "continued_tokens": len(o.token_ids)} |
| answer = text.split("</think>")[-1] if "</think>" in text else "" |
| if kind == "bench": |
| ok, pred = check(answer, gold[(rec["bench"], rec["id"])]["answer"], |
| gold[(rec["bench"], rec["id"])]["answer_type"]) |
| new.update(correct=bool(ok), pred=pred, thinking_closed="</think>" in text) |
| else: |
| g = gold[rec["key"]] |
| inp = InputExample(key=g["key"], instruction_id_list=g["instruction_id_list"], prompt=g["prompt"], |
| kwargs=g["kwargs"]) |
| strict = test_instruction_following_strict(inp, answer.strip()) |
| loose = test_instruction_following_loose(inp, answer.strip()) |
| new.update(strict_prompt=strict.follow_all_instructions, strict_inst=strict.follow_instruction_list, |
| loose_prompt=loose.follow_all_instructions, loose_inst=loose.follow_instruction_list) |
| out.append(new) |
|
|
| out_dir = Path(D) / "eval" / "results" / model.replace("/", "_") |
| out_dir.mkdir(parents=True, exist_ok=True) |
| name = f"{'ifeval_' if kind == 'ifeval' else ''}loops{loops}-cont{max_total // 1024}k.jsonl" |
| with open(out_dir / name, "w") as f: |
| for rec in out: |
| f.write(json.dumps(rec, ensure_ascii=False) + "\n") |
| data.commit() |
| summary = {"model": model, "loops": loops, "kind": kind, "n": len(out), "gen_s": round(gen_s), |
| "still_truncated": sum(r["finish"] == "length" for r in out), |
| "retok_diff_max": max(map(abs, retok), default=0), "prefix_tail": prefix[-40:]} |
| print("SUMMARY", json.dumps(summary), flush=True) |
| return summary |
|
|
|
|
| @APP.local_entrypoint() |
| def main(models: str = "base", loops: str = "2", limit: int = 0, medium_effort: bool = False, benches: str = ""): |
| configs = [(m, int(l)) for m in models.split(",") for l in loops.split(",")] |
| results = list(evaluate.starmap([(m, l, limit, 32768, medium_effort, benches) for m, l in configs])) |
| Path("data/eval").mkdir(parents=True, exist_ok=True) |
| stamp = time.strftime("%Y%m%d-%H%M%S") |
| Path(f"data/eval/summary-{stamp}.json").write_text(json.dumps(results, indent=1)) |
| for s in results: |
| line = " ".join(f"{b}={v['acc']:.3f}/{v['mean_tokens']}t" for b, v in s["benches"].items()) |
| print(f"{s['model']:>24s} loops={s['loops']}: {line}") |
|
|
