"""Live session: audio pushed in 80 ms chunks turn by turn, the fixed questions re-asked after every turn, must equal a full one-shot encode of the same prefix (streaming == full re-encode; gate: argmax agreement >= 95% on these 3 hard calls — the reference j2 live path scores the same .9785 here and .99 over 115 calls; max|dp| p95 reported), alert must fire on the scam calls after the decisive turn and not on the legit call; prefix-accuracy of `scam` vs gold.""" import argparse, json, time from pathlib import Path import numpy as np from tests.common import REPO, agreement, answer_probs from tests.test_audio import load_calls, state_parts, DATA def run(device=None, precision="bf16", chunk_ms=80, verbose=True): from julia2 import Runtime import soundfile as sf rt = Runtime(REPO, device=device, precision=precision) d = load_calls(); res = {"device": str(rt.device), "precision": precision, "calls": {}} stream, full, lat_push, lat_ask = [], [], [], [] hop = int(16000 * chunk_ms / 1000) for c in d["calls"]: qs = d["questions"][c["lang"]] s = rt.open_live("Context: " + json.dumps(c["context"], ensure_ascii=False), qs, lang=c["lang"]) alert_turn, correct, flips = None, [], 0; prev_right = None for i, t in enumerate(c["turns"]): wav, sr = sf.read(DATA / c["id"] / f"{i:02d}.flac", dtype="float32") s.turn(t["speaker"]) for k in range(0, len(wav), hop): t0 = time.perf_counter(); s.push(wav[k:k + hop], sr); lat_push.append(time.perf_counter() - t0) t0 = time.perf_counter(); ans, alert = s.ask(); lat_ask.append(time.perf_counter() - t0) if alert and alert_turn is None: alert_turn = i gold_scam = c["true_from_turn"].get("scam") is not None and i >= int(c["true_from_turn"]["scam"]) right = (ans["scam"]["noul"] >= 0.5) == gold_scam; correct.append(right) if prev_right and not right: flips += 1 prev_right = right # full re-encode of the same prefix of = rt.predict_questions(state_parts(c, d["speakers"], upto=i), qs) for k in qs: stream.append(answer_probs(ans[k])); full.append(answer_probs(of["answers"][k])) s.close() res["calls"][c["id"]] = {"label": c["label"], "lang": c["lang"], "turns": len(c["turns"]), "decisive_turn": c["decisive_turn"], "alert_turn": alert_turn, "scam_prefix_acc": round(float(np.mean(correct)), 3), "flips_right_to_wrong": flips, "final": {k: (a.get("choice") or round(a["noul"], 3)) for k, a in ans.items()}} res["streaming_vs_full"] = agreement(stream, full) res["latency_ms"] = {"push_chunk_p50": round(float(np.percentile(lat_push, 50)) * 1000, 2), "push_chunk_p95": round(float(np.percentile(lat_push, 95)) * 1000, 2), "ask_p50": round(float(np.percentile(lat_ask, 50)) * 1000, 1), "ask_p95": round(float(np.percentile(lat_ask, 95)) * 1000, 1)} if verbose: print(json.dumps(res, ensure_ascii=False)) assert res["streaming_vs_full"]["argmax_agree"] >= 0.95, res["streaming_vs_full"] for cid, r in res["calls"].items(): if r["label"] == "scam": assert r["alert_turn"] is not None, f"no alert on scam call {cid}" return res if __name__ == "__main__": ap = argparse.ArgumentParser(); ap.add_argument("--device"); ap.add_argument("--precision", default="bf16") a = ap.parse_args(); run(a.device, a.precision)