Download eval.py from debudash/slm-training: direct link, hf CLI and curl.
- Browser
- Download file 6.94 kB
-
https://huggingface.co/debudash/slm-training/resolve/main/eval.py
- Command line
-
hf download hf://debudash/slm-training/eval.py
-
curl -L -o eval.py https://huggingface.co/debudash/slm-training/resolve/main/eval.py
6.94 kB
| """One-command evaluation: generate replies with vLLM on Modal, judge every turn, print the results table. | |
| --model is a Hugging Face repo: the base model, or one of our LoRA adapter repos (detected by adapter_config.json, | |
| loaded on top of the pinned base). Multi-turn scenarios feed back the model's own replies. | |
| Prompt: "spec" gives the full spec and tables (the well-prompted base); "short" gives the one-line system prompt the | |
| tuned model was trained with. "auto" picks short for adapters and spec otherwise. | |
| Usage: python eval.py --model debudash/qwen3-4b-tz-convert-v1-n2000 --eval-set cases/ceiling.jsonl | |
| python eval.py --model Qwen/Qwen3-4B-Instruct-2507 --eval-set cases/ceiling.jsonl --prompt spec | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| import modal | |
| BASE_MODEL = "Qwen/Qwen3-4B-Instruct-2507" | |
| BASE_REVISION = "cdbee75f17c01a7cc42f958dc650907174af0554" | |
| # The FlashInfer sampler JIT-compiles with nvcc, which the slim image lacks; greedy decoding doesn't need it. | |
| image = (modal.Image.debian_slim(python_version="3.12").pip_install("vllm==0.31.0", "huggingface_hub") | |
| .env({"VLLM_USE_FLASHINFER_SAMPLER": "0"})) | |
| hf_cache = modal.Volume.from_name("hf-cache", create_if_missing=True) | |
| app = modal.App("slm-training-eval", image=image) | |
| def generate(model_id: str, revision: str | None, system: str, cases: list, max_tokens: int = 256): | |
| from huggingface_hub import HfApi, snapshot_download | |
| from vllm import LLM, SamplingParams | |
| from vllm.lora.request import LoRARequest | |
| api = HfApi() | |
| commit = api.model_info(model_id, revision=revision).sha | |
| adapter = "adapter_config.json" in api.list_repo_files(model_id, revision=revision) | |
| base, base_rev = (BASE_MODEL, BASE_REVISION) if adapter or model_id == BASE_MODEL else (model_id, commit) | |
| llm = LLM(model=base, revision=base_rev, enable_lora=adapter, max_lora_rank=64, max_model_len=8192, | |
| gpu_memory_utilization=0.85, seed=0) | |
| lora = LoRARequest("tuned", 1, snapshot_download(model_id, revision=commit)) if adapter else None | |
| params = SamplingParams(temperature=0, max_tokens=max_tokens) | |
| convs = [[{"role": "system", "content": system}] for _ in cases] | |
| replies = [[] for _ in cases] | |
| for t in range(max(len(c["turns"]) for c in cases)): # one batched wave per turn index | |
| live = [i for i, c in enumerate(cases) if t < len(c["turns"])] | |
| for i in live: | |
| convs[i].append({"role": "user", "content": cases[i]["turns"][t]["user"]}) | |
| outs = llm.chat([convs[i] for i in live], params, lora_request=lora, use_tqdm=False) | |
| for i, o in zip(live, outs): | |
| text = o.outputs[0].text | |
| replies[i].append(text) | |
| convs[i].append({"role": "assistant", "content": text or "(no reply)"}) | |
| return {"replies": replies, "commit": commit, "adapter": adapter, "base": base, "base_revision": base_rev} | |
| def drop_empty_modal_tokens(environ): | |
| """ISSUE-001: every eval failed with Modal AuthError "Token missing" once eval.py imported extract.py. | |
| Root cause: extract.py calls load_dotenv() before the Modal client starts, and .env (from the old | |
| .env.example) has empty MODAL_TOKEN_ID= / MODAL_TOKEN_SECRET= lines; Modal treats the empty variables as the | |
| credentials and ignores the `modal setup` login in ~/.modal.toml. Removing empty values lets it fall back.""" | |
| for key in ("MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"): | |
| if environ.get(key) == "": | |
| del environ[key] | |
| def system_prompt(kind): | |
| if kind == "short": | |
| from gen_train import SYSTEM | |
| return SYSTEM | |
| from run_ceiling import system_for | |
| return system_for("zero_shot") | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--model", required=True, help="Hugging Face repo id (base model or LoRA adapter repo)") | |
| ap.add_argument("--eval-set", required=True, help="scenario JSONL (as written by gen_cases.py)") | |
| ap.add_argument("--revision", default=None, help="repo revision; default is the current commit") | |
| ap.add_argument("--prompt", choices=["auto", "spec", "short"], default="auto") | |
| ap.add_argument("--label", default=None, help="results folder name; default derived from the model id") | |
| ap.add_argument("--out", default="results/eval") | |
| args = ap.parse_args() | |
| # Our scenario format, or plain prompts / messages; turns without an expected reply are labelled by | |
| # extract.py (the LLM copies components, the oracle computes the reply). | |
| from extract import label_missing, load_eval_set | |
| drop_empty_modal_tokens(os.environ) | |
| cases = label_missing(load_eval_set(args.eval_set)) | |
| extracted = sum(t.get("label_source") == "extracted" for c in cases for t in c["turns"]) | |
| if extracted: | |
| print(f"{extracted} turns had no expected reply; labelled by extraction + oracle") | |
| label = args.label or args.model.split("/")[-1] | |
| with modal.enable_output(), app.run(): | |
| prompt = args.prompt | |
| if prompt == "auto": | |
| prompt = "spec" if args.model == BASE_MODEL else "short" | |
| res = generate.remote(args.model, args.revision, system_prompt(prompt), cases) | |
| rows = [] | |
| for case, replies in zip(cases, res["replies"]): | |
| for i, (turn, text) in enumerate(zip(case["turns"], replies)): | |
| rows.append({"id": case["id"], "slice": case["slice"], "metric": case["metric"], "turn": i, | |
| "user": turn["user"], "expected": turn["expected"], "reply": text, | |
| "exact_match": text.strip() == turn["expected"], | |
| "label_source": turn.get("label_source", "given")}) | |
| out = Path(args.out) / label | |
| out.mkdir(parents=True, exist_ok=True) | |
| set_name = Path(args.eval_set).stem | |
| path = out / f"{label}__{prompt}-{set_name}__r1.jsonl" | |
| path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8") | |
| meta = {"model": args.model, "commit": res["commit"], "adapter": res["adapter"], "base": res["base"], | |
| "base_revision": res["base_revision"], "prompt": prompt, "eval_set": args.eval_set} | |
| (out / f"meta-{prompt}-{set_name}.json").write_text(json.dumps(meta, indent=1), encoding="utf-8") | |
| em = sum(r["exact_match"] for r in rows) / len(rows) | |
| print(f"{args.model}@{res['commit'][:8]} [{prompt}] on {args.eval_set}: {len(rows)} turns, exact match {em:.1%}") | |
| from judge import judge_file | |
| (out / "judged" / path.name).unlink(missing_ok=True) | |
| judge_file(path) | |
| subprocess.run([sys.executable, "report_ceiling.py", str(out / "judged")], check=True) | |
| if __name__ == "__main__": | |
| main() | |