slm-training / scripts /demo_parity.py
debudash's picture
Mirror slm-training @ 507b432 (course PDF excluded)
dc23a98 verified
Raw History Blame Contribute Delete
2.02 kB
"""Parity check: the demo endpoint must give byte-identical replies to the recorded eval runs.
Replays the first turn of the first 5 scenarios from each recorded transcript through serve_demo.py's endpoint:
the base with the spec prompt, tuned v3 (the demo's "before" model) and, given --final <label>, the final tuned model
against its own ceiling transcript in results/eval/<label>/. If they match, the demo shows what the evals measured.
Usage: python scripts/demo_parity.py [--final v5-n18000] (needs DEMO_URL and DEMO_TOKEN in .env)
"""
import json
import os
import sys
import requests
from dotenv import load_dotenv
load_dotenv()
URL, TOKEN = os.environ["DEMO_URL"].rstrip("/"), os.environ["DEMO_TOKEN"]
RECORDED = {"before": "results/eval/v3-n12000/judged/v3-n12000__short-ceiling__r1.jsonl",
"base": "results/eval/base/judged/base__spec-ceiling__r1.jsonl"}
if "--final" in sys.argv:
label = sys.argv[sys.argv.index("--final") + 1]
RECORDED["tuned"] = f"results/eval/{label}/judged/{label}__short-ceiling__r1.jsonl"
N = 5
def main():
fails = 0
print(f"{'model':<7} {'scenario':<26} {'match':<6} recorded | demo")
for model, path in RECORDED.items():
rows = [json.loads(l) for l in open(path, encoding="utf-8")]
firsts = [r for r in rows if r["turn"] == 0][:N]
for r in firsts:
resp = requests.post(f"{URL}/chat", json={"model": model, "messages": [{"role": "user", "content": r["user"]}]},
headers={"Authorization": f"Bearer {TOKEN}"}, timeout=900)
resp.raise_for_status()
reply = resp.json()["reply"]
ok = reply == r["reply"]
fails += not ok
print(f"{model:<7} {r['id']:<26} {'PASS' if ok else 'FAIL':<6} {r['reply'][:40]!r} | {reply[:40]!r}")
print(f"\n{N * len(RECORDED) - fails}/{N * len(RECORDED)} byte-identical")
sys.exit(1 if fails else 0)
if __name__ == "__main__":
main()