"""One-command evaluation: generate replies with vLLM on Modal, judge every turn, print the results table. --model is a Hugging Face repo: the base model, or one of our LoRA adapter repos (detected by adapter_config.json, loaded on top of the pinned base). Multi-turn scenarios feed back the model's own replies. Prompt: "spec" gives the full spec and tables (the well-prompted base); "short" gives the one-line system prompt the tuned model was trained with. "auto" picks short for adapters and spec otherwise. Usage: python eval.py --model debudash/qwen3-4b-tz-convert-v1-n2000 --eval-set cases/ceiling.jsonl python eval.py --model Qwen/Qwen3-4B-Instruct-2507 --eval-set cases/ceiling.jsonl --prompt spec """ import argparse import json import os import subprocess import sys from pathlib import Path import modal BASE_MODEL = "Qwen/Qwen3-4B-Instruct-2507" BASE_REVISION = "cdbee75f17c01a7cc42f958dc650907174af0554" # The FlashInfer sampler JIT-compiles with nvcc, which the slim image lacks; greedy decoding doesn't need it. image = (modal.Image.debian_slim(python_version="3.12").pip_install("vllm==0.31.0", "huggingface_hub") .env({"VLLM_USE_FLASHINFER_SAMPLER": "0"})) hf_cache = modal.Volume.from_name("hf-cache", create_if_missing=True) app = modal.App("slm-training-eval", image=image) @app.function(gpu="L40S", timeout=3600, volumes={"/root/.cache/huggingface": hf_cache}, secrets=[modal.Secret.from_name("huggingface")]) def generate(model_id: str, revision: str | None, system: str, cases: list, max_tokens: int = 256): from huggingface_hub import HfApi, snapshot_download from vllm import LLM, SamplingParams from vllm.lora.request import LoRARequest api = HfApi() commit = api.model_info(model_id, revision=revision).sha adapter = "adapter_config.json" in api.list_repo_files(model_id, revision=revision) base, base_rev = (BASE_MODEL, BASE_REVISION) if adapter or model_id == BASE_MODEL else (model_id, commit) llm = LLM(model=base, revision=base_rev, enable_lora=adapter, max_lora_rank=64, max_model_len=8192, gpu_memory_utilization=0.85, seed=0) lora = LoRARequest("tuned", 1, snapshot_download(model_id, revision=commit)) if adapter else None params = SamplingParams(temperature=0, max_tokens=max_tokens) convs = [[{"role": "system", "content": system}] for _ in cases] replies = [[] for _ in cases] for t in range(max(len(c["turns"]) for c in cases)): # one batched wave per turn index live = [i for i, c in enumerate(cases) if t < len(c["turns"])] for i in live: convs[i].append({"role": "user", "content": cases[i]["turns"][t]["user"]}) outs = llm.chat([convs[i] for i in live], params, lora_request=lora, use_tqdm=False) for i, o in zip(live, outs): text = o.outputs[0].text replies[i].append(text) convs[i].append({"role": "assistant", "content": text or "(no reply)"}) return {"replies": replies, "commit": commit, "adapter": adapter, "base": base, "base_revision": base_rev} def drop_empty_modal_tokens(environ): """ISSUE-001: every eval failed with Modal AuthError "Token missing" once eval.py imported extract.py. Root cause: extract.py calls load_dotenv() before the Modal client starts, and .env (from the old .env.example) has empty MODAL_TOKEN_ID= / MODAL_TOKEN_SECRET= lines; Modal treats the empty variables as the credentials and ignores the `modal setup` login in ~/.modal.toml. Removing empty values lets it fall back.""" for key in ("MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"): if environ.get(key) == "": del environ[key] def system_prompt(kind): if kind == "short": from gen_train import SYSTEM return SYSTEM from run_ceiling import system_for return system_for("zero_shot") def main(): ap = argparse.ArgumentParser() ap.add_argument("--model", required=True, help="Hugging Face repo id (base model or LoRA adapter repo)") ap.add_argument("--eval-set", required=True, help="scenario JSONL (as written by gen_cases.py)") ap.add_argument("--revision", default=None, help="repo revision; default is the current commit") ap.add_argument("--prompt", choices=["auto", "spec", "short"], default="auto") ap.add_argument("--label", default=None, help="results folder name; default derived from the model id") ap.add_argument("--out", default="results/eval") args = ap.parse_args() # Our scenario format, or plain prompts / messages; turns without an expected reply are labelled by # extract.py (the LLM copies components, the oracle computes the reply). from extract import label_missing, load_eval_set drop_empty_modal_tokens(os.environ) cases = label_missing(load_eval_set(args.eval_set)) extracted = sum(t.get("label_source") == "extracted" for c in cases for t in c["turns"]) if extracted: print(f"{extracted} turns had no expected reply; labelled by extraction + oracle") label = args.label or args.model.split("/")[-1] with modal.enable_output(), app.run(): prompt = args.prompt if prompt == "auto": prompt = "spec" if args.model == BASE_MODEL else "short" res = generate.remote(args.model, args.revision, system_prompt(prompt), cases) rows = [] for case, replies in zip(cases, res["replies"]): for i, (turn, text) in enumerate(zip(case["turns"], replies)): rows.append({"id": case["id"], "slice": case["slice"], "metric": case["metric"], "turn": i, "user": turn["user"], "expected": turn["expected"], "reply": text, "exact_match": text.strip() == turn["expected"], "label_source": turn.get("label_source", "given")}) out = Path(args.out) / label out.mkdir(parents=True, exist_ok=True) set_name = Path(args.eval_set).stem path = out / f"{label}__{prompt}-{set_name}__r1.jsonl" path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8") meta = {"model": args.model, "commit": res["commit"], "adapter": res["adapter"], "base": res["base"], "base_revision": res["base_revision"], "prompt": prompt, "eval_set": args.eval_set} (out / f"meta-{prompt}-{set_name}.json").write_text(json.dumps(meta, indent=1), encoding="utf-8") em = sum(r["exact_match"] for r in rows) / len(rows) print(f"{args.model}@{res['commit'][:8]} [{prompt}] on {args.eval_set}: {len(rows)} turns, exact match {em:.1%}") from judge import judge_file (out / "judged" / path.name).unlink(missing_ok=True) judge_file(path) subprocess.run([sys.executable, "report_ceiling.py", str(out / "judged")], check=True) if __name__ == "__main__": main()