xevyo-01 / code /eval /run_eval.py
meet4150's picture
xevyo-01 round 4: merged weights, adapter, model card, code
a46f204 verified
Raw History Blame Contribute Delete
8.94 kB
"""Ask one of OUR endpoints for a turn plan on every gold row, then score it.
python3 eval/run_eval.py --base http://127.0.0.1:18220 --model xevyo-01 \
--gold 'eval/gold/gold_*.jsonl' --out /data/xevyo-01/eval/xevyo-01_r1 --concurrency 8
Every request is what serving will send: the student's own prompt,
thinking off, temperature 0, and the per-turn schema through serve/vllm_schema.py
(response_format json_schema, the method that works on vLLM 0.17). Each answer is parsed,
checked with spec/validate.py and scored with eval/score.py.
Baselines through the same script:
* base Qwen3.5-9B zero-shot: --base <port serving the base> --model <name>
* a larger prompted model: --base <url serving it> --model <name>
--system-prefix-file <rules.md>
(the file is put in front of the student's system prompt, so the larger model
knows the rules the labels follow; the student never sees it)
Latency: --latency-pass N sends the first N rows again one at a time (concurrency 1)
and records their wall time; the scored latency comes from that pass when given.
The gold rows carry many different offered-tool sets (448 schemas in 1,610 rows, 93 in
the first 200), and vLLM 0.17 compiles a schema it has not cached on the request path, so that pass includes compile time on about half its
rows. --latency-same-tools N adds a second sequential pass over N rows that all share
the most common schema (335 gold rows have it; compiled once): the steady state of a
user whose tools do not change. Its p50 / p90 go into the report's meta.
Point this at your own xevyo-01 endpoint, never at a live service's model.
Standard library + jsonschema.
"""
import argparse
import json
import sys
import threading
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "spec"))
sys.path.insert(0, str(ROOT / "serve"))
sys.path.insert(0, str(ROOT / "eval"))
import render
import score
import validate
from vllm_schema import response_format
def post(base, body, timeout):
req = urllib.request.Request(base.rstrip("/") + "/v1/chat/completions", data=json.dumps(body).encode(),
headers={"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.loads(r.read())
def ask(args, row, prefix):
example = row["input"]
# Round 4: the context window; a row's own "window" wins over --window.
messages = render.render_messages(example, row.get("window") or args.window)
if prefix:
messages[0] = {"role": "system", "content": prefix + "\n\n---\n\n" + messages[0]["content"]}
body = {"model": args.model, "messages": messages, "temperature": 0, "max_tokens": args.max_tokens,
"response_format": response_format(example["offered_tools"]),
"chat_template_kwargs": {"enable_thinking": False}}
out = {"id": row["id"], "plan": None, "raw": None, "errors": [], "latency_s": None}
for attempt in range(3):
t0 = time.time()
try:
resp = post(args.base, body, args.timeout)
except (urllib.error.URLError, TimeoutError, ConnectionError) as e:
out["errors"] = [f"http: {e}"]
time.sleep(2 * (attempt + 1))
continue
out["latency_s"] = time.time() - t0
text = resp["choices"][0]["message"].get("content") or ""
out["raw"] = text
out["usage"] = resp.get("usage")
try:
plan = json.loads(text)
except json.JSONDecodeError as e:
out["errors"] = [f"json: {e}"]
return out
out["plan"] = plan
out["errors"] = validate.validate_plan(plan, example)
return out
return out
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--base", required=True, help="e.g. http://127.0.0.1:18220 (your xevyo-01 endpoint)")
ap.add_argument("--model", required=True)
ap.add_argument("--gold", nargs="+", required=True)
ap.add_argument("--out", required=True, help="output prefix (directory + name)")
ap.add_argument("--concurrency", type=int, default=8)
ap.add_argument("--max-tokens", type=int, default=512)
ap.add_argument("--timeout", type=float, default=120)
ap.add_argument("--limit", type=int, default=0)
ap.add_argument("--system-prefix-file", default="")
ap.add_argument("--latency-pass", type=int, default=0)
ap.add_argument("--latency-same-tools", type=int, default=0,
help="second sequential pass over N rows that share the most common offered-tool set")
ap.add_argument("--title", default="")
ap.add_argument("--window", default="trained", choices=render.WINDOWS,
help="context window the inputs are rendered in (the backend's XEVYO01_CONTEXT); default trained")
ap.add_argument("--no-score", action="store_true", help="only write predictions (rows without a target)")
args = ap.parse_args(argv)
port = args.base.rstrip("/").rsplit(":", 1)[-1]
if not (port.isdigit() and 18200 <= int(port) <= 18299):
sys.exit(f"refusing {args.base}: only our own ports 18200-18299")
gold = score.load_jsonl(args.gold)
if args.limit:
gold = gold[:args.limit]
prefix = Path(args.system_prefix_file).read_text(encoding="utf-8") if args.system_prefix_file else ""
out = Path(args.out)
out.parent.mkdir(parents=True, exist_ok=True)
pred_path = Path(str(out) + "_predictions.jsonl")
lock = threading.Lock()
done = [0]
t_start = time.time()
def job(row):
res = ask(args, row, prefix)
with lock:
done[0] += 1
if done[0] % 100 == 0:
print(f"{done[0]}/{len(gold)} in {time.time() - t_start:.0f}s", flush=True)
return res
with ThreadPoolExecutor(args.concurrency) as pool:
preds = list(pool.map(job, gold))
wall = time.time() - t_start
if args.latency_pass:
seq = {}
for row in gold[:args.latency_pass]:
r = ask(args, row, prefix)
seq[row["id"]] = r["latency_s"]
for p in preds:
p["latency_s"] = seq.get(p["id"])
same_tools = None
if args.latency_same_tools:
# Rows are grouped by the schema that is sent (what the server's grammar cache keys on),
# not by the raw tool list: lists that differ only in tools a plan cannot call share one.
key = lambda row: json.dumps(response_format(row["input"]["offered_tools"]), sort_keys=True)
counts = {}
for row in gold:
counts[key(row)] = counts.get(key(row), 0) + 1
common = max(counts, key=counts.get)
rows_same = [row for row in gold if key(row) == common][:args.latency_same_tools]
lat = sorted(x for x in (ask(args, row, prefix)["latency_s"] for row in rows_same) if x is not None)
if lat:
same_tools = {"rows": len(lat), "p50_s": round(lat[len(lat) // 2], 4),
"p90_s": round(lat[min(len(lat) - 1, int(len(lat) * 0.9))], 4),
"mean_s": round(sum(lat) / len(lat), 4)}
with open(pred_path, "w", encoding="utf-8") as f:
for p in preds:
f.write(json.dumps(p, ensure_ascii=False) + "\n")
if args.no_score:
print(f"wrote {pred_path} ({len(preds)} rows, {wall:.0f}s)")
return 0
report, rows, missing = score.score(gold, preds)
meta = {"base": args.base, "model": args.model, "gold": args.gold, "rows": len(gold),
"concurrency": args.concurrency, "wall_s": round(wall, 1),
"system_prefix_file": args.system_prefix_file or None,
"latency": f"sequential pass over {args.latency_pass} rows" if args.latency_pass else f"under concurrency {args.concurrency}",
"latency_same_tools": same_tools,
"unparsed": sum(1 for p in preds if p["plan"] is None),
"invalid": sum(1 for p in preds if p["plan"] is not None and p["errors"])}
traffic = score.traffic_weighted(rows) # TRAFFIC_WEIGHTS mix, see eval/score.py
Path(str(out) + ".json").write_text(json.dumps({"meta": meta, "report": report, "traffic": traffic}, indent=1),
encoding="utf-8")
with open(str(out) + "_rows.jsonl", "w", encoding="utf-8") as f:
for r in rows:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
md = score.markdown(report, args.title or f"{args.model} on {len(gold)} gold rows")
md += "\n" + score.traffic_markdown(traffic)
md += f"\n{json.dumps(meta)}\n"
Path(str(out) + ".md").write_text(md, encoding="utf-8")
print(md)
return 0
if __name__ == "__main__":
sys.exit(main())