File size: 4,582 Bytes
994182c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 | #!/usr/bin/env python3
"""Score a model on the held-out eval sets via in-process HF generate (no vLLM).
Useful when vLLM can't (yet) serve a model but the HF forward path works — e.g. the
Qwen3.6-27B base baseline. Emits the SAME report format as eval_endpoint.py
(reports/eval/<label>_eval.{json,md}) so numbers stay comparable across the base
model, in-training checkpoints, and any later vLLM-served runs.
Inference uses only the forward pass, so it is unaffected by the FLA gated-delta
*backward* kernel issue that blocks training on Hopper.
Example (on the GPU host, base model):
python training/scripts/eval_hf_model.py --model Qwen/Qwen3.6-27B --label base \
--eval data/eval/vuln_detection_test.jsonl --eval data/eval/knowledge_mcq.jsonl \
--report-dir reports/eval --sample 100
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from intraining_eval import run_eval_sets # noqa: E402
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--model", default="Qwen/Qwen3.6-27B")
p.add_argument("--eval", action="append", required=True, help="Eval JSONL (repeatable).")
p.add_argument("--label", required=True)
p.add_argument("--report-dir", default="reports/eval")
p.add_argument("--sample", type=int, default=100, help="Rows per set (default 100).")
p.add_argument("--max-new-tokens", type=int, default=256)
p.add_argument("--enable-thinking", action="store_true",
help="Let the model emit a <think> block first (slow; off by default for a fast tracking eval).")
p.add_argument("--dtype", default="bfloat16", choices=["bfloat16", "float16", "float32", "auto"])
p.add_argument("--device-map", default="auto")
return p.parse_args()
def load_model(model_name: str, dtype: str, device_map: str):
import torch
import transformers
dt = {"bfloat16": torch.bfloat16, "float16": torch.float16,
"float32": torch.float32, "auto": "auto"}[dtype]
errors = []
for cls_name in ["AutoModelForMultimodalLM", "AutoModelForImageTextToText",
"AutoModelForVision2Seq", "AutoModelForCausalLM"]:
cls = getattr(transformers, cls_name, None)
if cls is None:
continue
try:
model = cls.from_pretrained(model_name, dtype=dt, device_map=device_map, trust_remote_code=True)
print(f"loaded with {cls_name}")
return model
except Exception as exc: # noqa: BLE001
errors.append(f"{cls_name}: {exc!r}")
raise RuntimeError("could not load model:\n" + "\n".join(errors))
def main() -> int:
args = parse_args()
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
if tokenizer.pad_token_id is None:
tokenizer.pad_token = tokenizer.eos_token
model = load_model(args.model, args.dtype, args.device_map)
sets = run_eval_sets(model, tokenizer, args.eval, sample_per_set=args.sample,
max_new_tokens=args.max_new_tokens, enable_thinking=args.enable_thinking)
results = list(sets.values())
report_dir = Path(args.report_dir)
report_dir.mkdir(parents=True, exist_ok=True)
payload = {"label": args.label, "model": args.model, "mock": False, "via": "hf_generate", "results": results}
(report_dir / f"{args.label}_eval.json").write_text(json.dumps(payload, indent=2), encoding="utf-8")
lines = [f"# Endpoint Eval — {args.label} (HF generate)", "", f"- Model: `{args.model}`", ""]
for r in results:
if "error" in r:
lines.append(f"## ERROR: {r}")
continue
lines.append(f"## {r.get('kind')}")
lines.append(f"- n: {r.get('n')} accuracy: **{r.get('accuracy', 0):.2%}**")
if r.get("kind") == "vuln_detection":
lines.append(f"- precision(vuln): {r.get('precision_vuln', 0):.2%} "
f"recall(vuln): {r.get('recall_vuln', 0):.2%} f1: {r.get('f1_vuln', 0):.2%} "
f"unparsed: {r.get('unparsed')}")
else:
lines.append(f"- unparsed: {r.get('unparsed')}")
lines.append("")
(report_dir / f"{args.label}_eval.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
print(json.dumps(payload, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
|