File size: 6,937 Bytes
4643f62
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
"""One-command evaluation: generate replies with vLLM on Modal, judge every turn, print the results table.



--model is a Hugging Face repo: the base model, or one of our LoRA adapter repos (detected by adapter_config.json,

loaded on top of the pinned base). Multi-turn scenarios feed back the model's own replies.

Prompt: "spec" gives the full spec and tables (the well-prompted base); "short" gives the one-line system prompt the

tuned model was trained with. "auto" picks short for adapters and spec otherwise.



Usage: python eval.py --model debudash/qwen3-4b-tz-convert-v1-n2000 --eval-set cases/ceiling.jsonl

       python eval.py --model Qwen/Qwen3-4B-Instruct-2507 --eval-set cases/ceiling.jsonl --prompt spec

"""
import argparse
import json
import os
import subprocess
import sys
from pathlib import Path

import modal

BASE_MODEL = "Qwen/Qwen3-4B-Instruct-2507"
BASE_REVISION = "cdbee75f17c01a7cc42f958dc650907174af0554"

# The FlashInfer sampler JIT-compiles with nvcc, which the slim image lacks; greedy decoding doesn't need it.
image = (modal.Image.debian_slim(python_version="3.12").pip_install("vllm==0.31.0", "huggingface_hub")
         .env({"VLLM_USE_FLASHINFER_SAMPLER": "0"}))
hf_cache = modal.Volume.from_name("hf-cache", create_if_missing=True)
app = modal.App("slm-training-eval", image=image)


@app.function(gpu="L40S", timeout=3600, volumes={"/root/.cache/huggingface": hf_cache},

              secrets=[modal.Secret.from_name("huggingface")])
def generate(model_id: str, revision: str | None, system: str, cases: list, max_tokens: int = 256):
    from huggingface_hub import HfApi, snapshot_download
    from vllm import LLM, SamplingParams
    from vllm.lora.request import LoRARequest

    api = HfApi()
    commit = api.model_info(model_id, revision=revision).sha
    adapter = "adapter_config.json" in api.list_repo_files(model_id, revision=revision)
    base, base_rev = (BASE_MODEL, BASE_REVISION) if adapter or model_id == BASE_MODEL else (model_id, commit)
    llm = LLM(model=base, revision=base_rev, enable_lora=adapter, max_lora_rank=64, max_model_len=8192,
              gpu_memory_utilization=0.85, seed=0)
    lora = LoRARequest("tuned", 1, snapshot_download(model_id, revision=commit)) if adapter else None
    params = SamplingParams(temperature=0, max_tokens=max_tokens)

    convs = [[{"role": "system", "content": system}] for _ in cases]
    replies = [[] for _ in cases]
    for t in range(max(len(c["turns"]) for c in cases)):  # one batched wave per turn index
        live = [i for i, c in enumerate(cases) if t < len(c["turns"])]
        for i in live:
            convs[i].append({"role": "user", "content": cases[i]["turns"][t]["user"]})
        outs = llm.chat([convs[i] for i in live], params, lora_request=lora, use_tqdm=False)
        for i, o in zip(live, outs):
            text = o.outputs[0].text
            replies[i].append(text)
            convs[i].append({"role": "assistant", "content": text or "(no reply)"})
    return {"replies": replies, "commit": commit, "adapter": adapter, "base": base, "base_revision": base_rev}


def drop_empty_modal_tokens(environ):
    """ISSUE-001: every eval failed with Modal AuthError "Token missing" once eval.py imported extract.py.

    Root cause: extract.py calls load_dotenv() before the Modal client starts, and .env (from the old

    .env.example) has empty MODAL_TOKEN_ID= / MODAL_TOKEN_SECRET= lines; Modal treats the empty variables as the

    credentials and ignores the `modal setup` login in ~/.modal.toml. Removing empty values lets it fall back."""
    for key in ("MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"):
        if environ.get(key) == "":
            del environ[key]


def system_prompt(kind):
    if kind == "short":
        from gen_train import SYSTEM
        return SYSTEM
    from run_ceiling import system_for
    return system_for("zero_shot")


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--model", required=True, help="Hugging Face repo id (base model or LoRA adapter repo)")
    ap.add_argument("--eval-set", required=True, help="scenario JSONL (as written by gen_cases.py)")
    ap.add_argument("--revision", default=None, help="repo revision; default is the current commit")
    ap.add_argument("--prompt", choices=["auto", "spec", "short"], default="auto")
    ap.add_argument("--label", default=None, help="results folder name; default derived from the model id")
    ap.add_argument("--out", default="results/eval")
    args = ap.parse_args()

    # Our scenario format, or plain prompts / messages; turns without an expected reply are labelled by
    # extract.py (the LLM copies components, the oracle computes the reply).
    from extract import label_missing, load_eval_set
    drop_empty_modal_tokens(os.environ)
    cases = label_missing(load_eval_set(args.eval_set))
    extracted = sum(t.get("label_source") == "extracted" for c in cases for t in c["turns"])
    if extracted:
        print(f"{extracted} turns had no expected reply; labelled by extraction + oracle")
    label = args.label or args.model.split("/")[-1]
    with modal.enable_output(), app.run():
        prompt = args.prompt
        if prompt == "auto":
            prompt = "spec" if args.model == BASE_MODEL else "short"
        res = generate.remote(args.model, args.revision, system_prompt(prompt), cases)

    rows = []
    for case, replies in zip(cases, res["replies"]):
        for i, (turn, text) in enumerate(zip(case["turns"], replies)):
            rows.append({"id": case["id"], "slice": case["slice"], "metric": case["metric"], "turn": i,
                         "user": turn["user"], "expected": turn["expected"], "reply": text,
                         "exact_match": text.strip() == turn["expected"],
                         "label_source": turn.get("label_source", "given")})
    out = Path(args.out) / label
    out.mkdir(parents=True, exist_ok=True)
    set_name = Path(args.eval_set).stem
    path = out / f"{label}__{prompt}-{set_name}__r1.jsonl"
    path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8")
    meta = {"model": args.model, "commit": res["commit"], "adapter": res["adapter"], "base": res["base"],
            "base_revision": res["base_revision"], "prompt": prompt, "eval_set": args.eval_set}
    (out / f"meta-{prompt}-{set_name}.json").write_text(json.dumps(meta, indent=1), encoding="utf-8")
    em = sum(r["exact_match"] for r in rows) / len(rows)
    print(f"{args.model}@{res['commit'][:8]} [{prompt}] on {args.eval_set}: {len(rows)} turns, exact match {em:.1%}")

    from judge import judge_file
    (out / "judged" / path.name).unlink(missing_ok=True)
    judge_file(path)
    subprocess.run([sys.executable, "report_ceiling.py", str(out / "judged")], check=True)


if __name__ == "__main__":
    main()