Download scripts/demo_parity.py from debudash/slm-training: direct link, hf CLI and curl.
- Browser
- Download file 2.02 kB
-
https://huggingface.co/debudash/slm-training/resolve/main/scripts/demo_parity.py
- Command line
-
hf download hf://debudash/slm-training/scripts/demo_parity.py
-
curl -L -o demo_parity.py https://huggingface.co/debudash/slm-training/resolve/main/scripts/demo_parity.py
2.02 kB
| """Parity check: the demo endpoint must give byte-identical replies to the recorded eval runs. | |
| Replays the first turn of the first 5 scenarios from each recorded transcript through serve_demo.py's endpoint: | |
| the base with the spec prompt, tuned v3 (the demo's "before" model) and, given --final <label>, the final tuned model | |
| against its own ceiling transcript in results/eval/<label>/. If they match, the demo shows what the evals measured. | |
| Usage: python scripts/demo_parity.py [--final v5-n18000] (needs DEMO_URL and DEMO_TOKEN in .env) | |
| """ | |
| import json | |
| import os | |
| import sys | |
| import requests | |
| from dotenv import load_dotenv | |
| load_dotenv() | |
| URL, TOKEN = os.environ["DEMO_URL"].rstrip("/"), os.environ["DEMO_TOKEN"] | |
| RECORDED = {"before": "results/eval/v3-n12000/judged/v3-n12000__short-ceiling__r1.jsonl", | |
| "base": "results/eval/base/judged/base__spec-ceiling__r1.jsonl"} | |
| if "--final" in sys.argv: | |
| label = sys.argv[sys.argv.index("--final") + 1] | |
| RECORDED["tuned"] = f"results/eval/{label}/judged/{label}__short-ceiling__r1.jsonl" | |
| N = 5 | |
| def main(): | |
| fails = 0 | |
| print(f"{'model':<7} {'scenario':<26} {'match':<6} recorded | demo") | |
| for model, path in RECORDED.items(): | |
| rows = [json.loads(l) for l in open(path, encoding="utf-8")] | |
| firsts = [r for r in rows if r["turn"] == 0][:N] | |
| for r in firsts: | |
| resp = requests.post(f"{URL}/chat", json={"model": model, "messages": [{"role": "user", "content": r["user"]}]}, | |
| headers={"Authorization": f"Bearer {TOKEN}"}, timeout=900) | |
| resp.raise_for_status() | |
| reply = resp.json()["reply"] | |
| ok = reply == r["reply"] | |
| fails += not ok | |
| print(f"{model:<7} {r['id']:<26} {'PASS' if ok else 'FAIL':<6} {r['reply'][:40]!r} | {reply[:40]!r}") | |
| print(f"\n{N * len(RECORDED) - fails}/{N * len(RECORDED)} byte-identical") | |
| sys.exit(1 if fails else 0) | |
| if __name__ == "__main__": | |
| main() | |