slm-training / eval.py
debudash's picture
Mirror slm-training at 4376283
4643f62 verified
Raw History Blame Contribute Delete
6.94 kB
"""One-command evaluation: generate replies with vLLM on Modal, judge every turn, print the results table.
--model is a Hugging Face repo: the base model, or one of our LoRA adapter repos (detected by adapter_config.json,
loaded on top of the pinned base). Multi-turn scenarios feed back the model's own replies.
Prompt: "spec" gives the full spec and tables (the well-prompted base); "short" gives the one-line system prompt the
tuned model was trained with. "auto" picks short for adapters and spec otherwise.
Usage: python eval.py --model debudash/qwen3-4b-tz-convert-v1-n2000 --eval-set cases/ceiling.jsonl
python eval.py --model Qwen/Qwen3-4B-Instruct-2507 --eval-set cases/ceiling.jsonl --prompt spec
"""
import argparse
import json
import os
import subprocess
import sys
from pathlib import Path
import modal
BASE_MODEL = "Qwen/Qwen3-4B-Instruct-2507"
BASE_REVISION = "cdbee75f17c01a7cc42f958dc650907174af0554"
# The FlashInfer sampler JIT-compiles with nvcc, which the slim image lacks; greedy decoding doesn't need it.
image = (modal.Image.debian_slim(python_version="3.12").pip_install("vllm==0.31.0", "huggingface_hub")
.env({"VLLM_USE_FLASHINFER_SAMPLER": "0"}))
hf_cache = modal.Volume.from_name("hf-cache", create_if_missing=True)
app = modal.App("slm-training-eval", image=image)
@app.function(gpu="L40S", timeout=3600, volumes={"/root/.cache/huggingface": hf_cache},
secrets=[modal.Secret.from_name("huggingface")])
def generate(model_id: str, revision: str | None, system: str, cases: list, max_tokens: int = 256):
from huggingface_hub import HfApi, snapshot_download
from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest
api = HfApi()
commit = api.model_info(model_id, revision=revision).sha
adapter = "adapter_config.json" in api.list_repo_files(model_id, revision=revision)
base, base_rev = (BASE_MODEL, BASE_REVISION) if adapter or model_id == BASE_MODEL else (model_id, commit)
llm = LLM(model=base, revision=base_rev, enable_lora=adapter, max_lora_rank=64, max_model_len=8192,
gpu_memory_utilization=0.85, seed=0)
lora = LoRARequest("tuned", 1, snapshot_download(model_id, revision=commit)) if adapter else None
params = SamplingParams(temperature=0, max_tokens=max_tokens)
convs = [[{"role": "system", "content": system}] for _ in cases]
replies = [[] for _ in cases]
for t in range(max(len(c["turns"]) for c in cases)): # one batched wave per turn index
live = [i for i, c in enumerate(cases) if t < len(c["turns"])]
for i in live:
convs[i].append({"role": "user", "content": cases[i]["turns"][t]["user"]})
outs = llm.chat([convs[i] for i in live], params, lora_request=lora, use_tqdm=False)
for i, o in zip(live, outs):
text = o.outputs[0].text
replies[i].append(text)
convs[i].append({"role": "assistant", "content": text or "(no reply)"})
return {"replies": replies, "commit": commit, "adapter": adapter, "base": base, "base_revision": base_rev}
def drop_empty_modal_tokens(environ):
"""ISSUE-001: every eval failed with Modal AuthError "Token missing" once eval.py imported extract.py.
Root cause: extract.py calls load_dotenv() before the Modal client starts, and .env (from the old
.env.example) has empty MODAL_TOKEN_ID= / MODAL_TOKEN_SECRET= lines; Modal treats the empty variables as the
credentials and ignores the `modal setup` login in ~/.modal.toml. Removing empty values lets it fall back."""
for key in ("MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"):
if environ.get(key) == "":
del environ[key]
def system_prompt(kind):
if kind == "short":
from gen_train import SYSTEM
return SYSTEM
from run_ceiling import system_for
return system_for("zero_shot")
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--model", required=True, help="Hugging Face repo id (base model or LoRA adapter repo)")
ap.add_argument("--eval-set", required=True, help="scenario JSONL (as written by gen_cases.py)")
ap.add_argument("--revision", default=None, help="repo revision; default is the current commit")
ap.add_argument("--prompt", choices=["auto", "spec", "short"], default="auto")
ap.add_argument("--label", default=None, help="results folder name; default derived from the model id")
ap.add_argument("--out", default="results/eval")
args = ap.parse_args()
# Our scenario format, or plain prompts / messages; turns without an expected reply are labelled by
# extract.py (the LLM copies components, the oracle computes the reply).
from extract import label_missing, load_eval_set
drop_empty_modal_tokens(os.environ)
cases = label_missing(load_eval_set(args.eval_set))
extracted = sum(t.get("label_source") == "extracted" for c in cases for t in c["turns"])
if extracted:
print(f"{extracted} turns had no expected reply; labelled by extraction + oracle")
label = args.label or args.model.split("/")[-1]
with modal.enable_output(), app.run():
prompt = args.prompt
if prompt == "auto":
prompt = "spec" if args.model == BASE_MODEL else "short"
res = generate.remote(args.model, args.revision, system_prompt(prompt), cases)
rows = []
for case, replies in zip(cases, res["replies"]):
for i, (turn, text) in enumerate(zip(case["turns"], replies)):
rows.append({"id": case["id"], "slice": case["slice"], "metric": case["metric"], "turn": i,
"user": turn["user"], "expected": turn["expected"], "reply": text,
"exact_match": text.strip() == turn["expected"],
"label_source": turn.get("label_source", "given")})
out = Path(args.out) / label
out.mkdir(parents=True, exist_ok=True)
set_name = Path(args.eval_set).stem
path = out / f"{label}__{prompt}-{set_name}__r1.jsonl"
path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8")
meta = {"model": args.model, "commit": res["commit"], "adapter": res["adapter"], "base": res["base"],
"base_revision": res["base_revision"], "prompt": prompt, "eval_set": args.eval_set}
(out / f"meta-{prompt}-{set_name}.json").write_text(json.dumps(meta, indent=1), encoding="utf-8")
em = sum(r["exact_match"] for r in rows) / len(rows)
print(f"{args.model}@{res['commit'][:8]} [{prompt}] on {args.eval_set}: {len(rows)} turns, exact match {em:.1%}")
from judge import judge_file
(out / "judged" / path.name).unlink(missing_ok=True)
judge_file(path)
subprocess.run([sys.executable, "report_ceiling.py", str(out / "judged")], check=True)
if __name__ == "__main__":
main()