"""Precision/answer-agreement gate. Inputs are cached once (tests/gate_inputs.pt: token ids of 300 text rows, decoded audio of 100 TTS states / 354 questions, decoded audio of 115 held-out calls) and the bf16 outputs are saved once (tests/gate_ref_bf16.json); every variant compares against that file. python -m tools.gate --precision bf16 --save_ref # reference (also reports accuracy vs gold) python -m tools.gate --precision int8 # agreement vs the saved bf16 reference """ import argparse, json, sys, time from pathlib import Path import numpy as np, torch HERE = Path(__file__).resolve().parent; sys.path.insert(0, str(HERE.parent)) from tests.common import REPO, agreement, text_ref, to_rows, answer_probs CACHE = HERE.parent / "tests" / "gate_inputs.pt" J2 = Path("/home/ubuntu/julia-2") def build_inputs(): """Needs the j2 data (TTS packs, call renders). Cached to gate_inputs.pt (reused by every gate run).""" import soundfile as sf sys.path.insert(0, str(J2)) from j2.audio_train import Pack from j2.gen.mix3 import h as hsh from j2 import calls as C pack = Pack(J2 / "data/audio/packs/tts_scen"); val = [r for r in pack.idx if hsh(r["id"] + "-split") < 0.03][:100] scen, items, wavs = {}, [], {} for r in val: name, i = r["id"].rsplit("-", 1) if name not in scen: scen[name] = [json.loads(l) for l in open(J2 / f"data/gen/{name}.jsonl")] wavs[r["id"]] = torch.from_numpy(pack.wav(r)) for q in scen[name][int(i)]["questions"]: if q.get("verified", True): items.append({"id": r["id"], "type": q["type"], "question": q["question"], "options": [list(o) for o in q["options"]], "gold": int(np.argmax(q["target"]))}) meta = {json.loads(l)["id"]: json.loads(l) for l in open(J2 / "data/audio/calls/meta.jsonl")} calls = [c for c in C.load_calls(("calls_v2",), split="eval") if c["id"] in meta] cw = {} for c in calls: cw[c["id"]] = [torch.from_numpy(sf.read(J2 / f"data/audio/calls/{c['id']}/{i:02d}.flac", dtype="float32")[0]) for i in range(len(c["turns"]))] qs = {} for q in C.questions(c): qs[q["key"]] = {"type": "noul", "instructions": q["question"], "criteria": {"true": q["options"][0][1], "false": q["options"][1][1]}} if q["type"] == "noul" else {"type": "choice", "instructions": q["question"], "criteria": dict(q["options"])} c["sysq"] = qs; c["gold_final"] = C.gold_at(c, len(c["turns"]) - 1); c["context_text"] = C.context_text(c) return {"audio_items": items, "audio_wavs": wavs, "calls": calls, "call_wavs": cw} def main(): ap = argparse.ArgumentParser(); ap.add_argument("--precision", default="bf16"); ap.add_argument("--repo", default=str(REPO)); ap.add_argument("--save_ref", action="store_true"); ap.add_argument("--device"); ap.add_argument("--n_calls", type=int, default=115) a = ap.parse_args() if not CACHE.exists(): torch.save(build_inputs(), CACHE) inp = torch.load(CACHE, weights_only=False) from julia2 import Runtime t0 = time.time(); rt = Runtime(a.repo, device=a.device, precision=a.precision); res = {"precision": a.precision, "repo": a.repo, "load_s": round(time.time() - t0, 1)} out = {} ref = text_ref(); t0 = time.time(); out["text"], _ = rt._probs_text(to_rows(ref)); res["text_sec"] = round(time.time() - t0, 1) res["text_vs_j2ref"] = agreement(out["text"], [r["ref"] for r in ref]) t0 = time.time() out["audio"] = [rt._probs_shared([("audio", inp["audio_wavs"][it["id"]].numpy())], [{"type": it["type"], "question": it["question"], "options": [tuple(o) for o in it["options"]]}])[0] for it in inp["audio_items"]] res["audio_sec"] = round(time.time() - t0, 1); res["audio_acc"] = round(float(np.mean([np.argmax(p) == it["gold"] for p, it in zip(out["audio"], inp["audio_items"])])), 4) t0 = time.time(); live, right, alerts = [], [], [] for c in inp["calls"][: a.n_calls]: s = rt.open_live(c["context_text"], c["sysq"], lang=c["lang"]) for i, t in enumerate(c["turns"]): s.turn(t["speaker"]); s.push(inp["call_wavs"][c["id"]][i].numpy()); ans, alert = s.ask() for k, v in ans.items(): live.append(answer_probs(v)) if i == len(c["turns"]) - 1 and c["gold_final"][k] is not None: right.append(int(np.argmax(answer_probs(v)) == c["gold_final"][k])) alerts.append((c["label"], alert)); s.close() out["live"] = live; res["live_sec"] = round(time.time() - t0, 1); res["live_final_acc"] = round(float(np.mean(right)), 4) res["live_alert_recall_scam"] = round(float(np.mean([al for lb, al in alerts if lb == "scam"])), 4); res["live_false_alarm_legit"] = round(float(np.mean([al for lb, al in alerts if lb == "legit"])), 4) if rt.device.type == "cuda": res["peak_mem_gb"] = round(torch.cuda.max_memory_allocated() / 1e9, 2) refp = HERE.parent / "tests" / "gate_ref_bf16.json" if a.save_ref: json.dump(out, open(refp, "w")) elif refp.exists(): R = json.load(open(refp)) for k in ("text", "audio", "live"): res[f"{k}_vs_bf16"] = agreement(out[k], R[k]) print(json.dumps(res)) Path(a.repo, "metrics").mkdir(exist_ok=True) json.dump(res, open(Path(a.repo) / "metrics" / f"gate_{a.precision}.json", "w"), indent=1) if __name__ == "__main__": main()