"""Ask one of OUR endpoints for a turn plan on every gold row, then score it. python3 eval/run_eval.py --base http://127.0.0.1:18220 --model xevyo-01 \ --gold 'eval/gold/gold_*.jsonl' --out /data/xevyo-01/eval/xevyo-01_r1 --concurrency 8 Every request is what serving will send: the student's own prompt, thinking off, temperature 0, and the per-turn schema through serve/vllm_schema.py (response_format json_schema, the method that works on vLLM 0.17). Each answer is parsed, checked with spec/validate.py and scored with eval/score.py. Baselines through the same script: * base Qwen3.5-9B zero-shot: --base --model * a larger prompted model: --base --model --system-prefix-file (the file is put in front of the student's system prompt, so the larger model knows the rules the labels follow; the student never sees it) Latency: --latency-pass N sends the first N rows again one at a time (concurrency 1) and records their wall time; the scored latency comes from that pass when given. The gold rows carry many different offered-tool sets (448 schemas in 1,610 rows, 93 in the first 200), and vLLM 0.17 compiles a schema it has not cached on the request path, so that pass includes compile time on about half its rows. --latency-same-tools N adds a second sequential pass over N rows that all share the most common schema (335 gold rows have it; compiled once): the steady state of a user whose tools do not change. Its p50 / p90 go into the report's meta. Point this at your own xevyo-01 endpoint, never at a live service's model. Standard library + jsonschema. """ import argparse import json import sys import threading import time import urllib.error import urllib.request from concurrent.futures import ThreadPoolExecutor from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "spec")) sys.path.insert(0, str(ROOT / "serve")) sys.path.insert(0, str(ROOT / "eval")) import render import score import validate from vllm_schema import response_format def post(base, body, timeout): req = urllib.request.Request(base.rstrip("/") + "/v1/chat/completions", data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) with urllib.request.urlopen(req, timeout=timeout) as r: return json.loads(r.read()) def ask(args, row, prefix): example = row["input"] # Round 4: the context window; a row's own "window" wins over --window. messages = render.render_messages(example, row.get("window") or args.window) if prefix: messages[0] = {"role": "system", "content": prefix + "\n\n---\n\n" + messages[0]["content"]} body = {"model": args.model, "messages": messages, "temperature": 0, "max_tokens": args.max_tokens, "response_format": response_format(example["offered_tools"]), "chat_template_kwargs": {"enable_thinking": False}} out = {"id": row["id"], "plan": None, "raw": None, "errors": [], "latency_s": None} for attempt in range(3): t0 = time.time() try: resp = post(args.base, body, args.timeout) except (urllib.error.URLError, TimeoutError, ConnectionError) as e: out["errors"] = [f"http: {e}"] time.sleep(2 * (attempt + 1)) continue out["latency_s"] = time.time() - t0 text = resp["choices"][0]["message"].get("content") or "" out["raw"] = text out["usage"] = resp.get("usage") try: plan = json.loads(text) except json.JSONDecodeError as e: out["errors"] = [f"json: {e}"] return out out["plan"] = plan out["errors"] = validate.validate_plan(plan, example) return out return out def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--base", required=True, help="e.g. http://127.0.0.1:18220 (your xevyo-01 endpoint)") ap.add_argument("--model", required=True) ap.add_argument("--gold", nargs="+", required=True) ap.add_argument("--out", required=True, help="output prefix (directory + name)") ap.add_argument("--concurrency", type=int, default=8) ap.add_argument("--max-tokens", type=int, default=512) ap.add_argument("--timeout", type=float, default=120) ap.add_argument("--limit", type=int, default=0) ap.add_argument("--system-prefix-file", default="") ap.add_argument("--latency-pass", type=int, default=0) ap.add_argument("--latency-same-tools", type=int, default=0, help="second sequential pass over N rows that share the most common offered-tool set") ap.add_argument("--title", default="") ap.add_argument("--window", default="trained", choices=render.WINDOWS, help="context window the inputs are rendered in (the backend's XEVYO01_CONTEXT); default trained") ap.add_argument("--no-score", action="store_true", help="only write predictions (rows without a target)") args = ap.parse_args(argv) port = args.base.rstrip("/").rsplit(":", 1)[-1] if not (port.isdigit() and 18200 <= int(port) <= 18299): sys.exit(f"refusing {args.base}: only our own ports 18200-18299") gold = score.load_jsonl(args.gold) if args.limit: gold = gold[:args.limit] prefix = Path(args.system_prefix_file).read_text(encoding="utf-8") if args.system_prefix_file else "" out = Path(args.out) out.parent.mkdir(parents=True, exist_ok=True) pred_path = Path(str(out) + "_predictions.jsonl") lock = threading.Lock() done = [0] t_start = time.time() def job(row): res = ask(args, row, prefix) with lock: done[0] += 1 if done[0] % 100 == 0: print(f"{done[0]}/{len(gold)} in {time.time() - t_start:.0f}s", flush=True) return res with ThreadPoolExecutor(args.concurrency) as pool: preds = list(pool.map(job, gold)) wall = time.time() - t_start if args.latency_pass: seq = {} for row in gold[:args.latency_pass]: r = ask(args, row, prefix) seq[row["id"]] = r["latency_s"] for p in preds: p["latency_s"] = seq.get(p["id"]) same_tools = None if args.latency_same_tools: # Rows are grouped by the schema that is sent (what the server's grammar cache keys on), # not by the raw tool list: lists that differ only in tools a plan cannot call share one. key = lambda row: json.dumps(response_format(row["input"]["offered_tools"]), sort_keys=True) counts = {} for row in gold: counts[key(row)] = counts.get(key(row), 0) + 1 common = max(counts, key=counts.get) rows_same = [row for row in gold if key(row) == common][:args.latency_same_tools] lat = sorted(x for x in (ask(args, row, prefix)["latency_s"] for row in rows_same) if x is not None) if lat: same_tools = {"rows": len(lat), "p50_s": round(lat[len(lat) // 2], 4), "p90_s": round(lat[min(len(lat) - 1, int(len(lat) * 0.9))], 4), "mean_s": round(sum(lat) / len(lat), 4)} with open(pred_path, "w", encoding="utf-8") as f: for p in preds: f.write(json.dumps(p, ensure_ascii=False) + "\n") if args.no_score: print(f"wrote {pred_path} ({len(preds)} rows, {wall:.0f}s)") return 0 report, rows, missing = score.score(gold, preds) meta = {"base": args.base, "model": args.model, "gold": args.gold, "rows": len(gold), "concurrency": args.concurrency, "wall_s": round(wall, 1), "system_prefix_file": args.system_prefix_file or None, "latency": f"sequential pass over {args.latency_pass} rows" if args.latency_pass else f"under concurrency {args.concurrency}", "latency_same_tools": same_tools, "unparsed": sum(1 for p in preds if p["plan"] is None), "invalid": sum(1 for p in preds if p["plan"] is not None and p["errors"])} traffic = score.traffic_weighted(rows) # TRAFFIC_WEIGHTS mix, see eval/score.py Path(str(out) + ".json").write_text(json.dumps({"meta": meta, "report": report, "traffic": traffic}, indent=1), encoding="utf-8") with open(str(out) + "_rows.jsonl", "w", encoding="utf-8") as f: for r in rows: f.write(json.dumps(r, ensure_ascii=False) + "\n") md = score.markdown(report, args.title or f"{args.model} on {len(gold)} gold rows") md += "\n" + score.traffic_markdown(traffic) md += f"\n{json.dumps(meta)}\n" Path(str(out) + ".md").write_text(md, encoding="utf-8") print(md) return 0 if __name__ == "__main__": sys.exit(main())