Julia-2 / runtime /tests /test_live.py
Julia Klee
Julia-2: julia2_audio_v1 (text = base_v2 byte-identical, + pruned audio tower, stem p9), int8/int4 variants, lean runtime 0.1.0 (audio, LiveSession, modalities), card with speech, live calls and variants
1d1c6db
Raw History Blame Contribute Delete
3.51 kB
"""Live session: audio pushed in 80 ms chunks turn by turn, the fixed questions re-asked after every turn, must equal a
full one-shot encode of the same prefix (streaming == full re-encode; gate: argmax agreement >= 95% on these 3 hard calls — the
reference j2 live path scores the same .9785 here and .99 over 115 calls; max|dp| p95 reported),
alert must fire on the scam calls after the decisive turn and not on the legit call; prefix-accuracy of `scam` vs gold."""
import argparse, json, time
from pathlib import Path
import numpy as np
from tests.common import REPO, agreement, answer_probs
from tests.test_audio import load_calls, state_parts, DATA
def run(device=None, precision="bf16", chunk_ms=80, verbose=True):
from julia2 import Runtime
import soundfile as sf
rt = Runtime(REPO, device=device, precision=precision)
d = load_calls(); res = {"device": str(rt.device), "precision": precision, "calls": {}}
stream, full, lat_push, lat_ask = [], [], [], []
hop = int(16000 * chunk_ms / 1000)
for c in d["calls"]:
qs = d["questions"][c["lang"]]
s = rt.open_live("Context: " + json.dumps(c["context"], ensure_ascii=False), qs, lang=c["lang"])
alert_turn, correct, flips = None, [], 0; prev_right = None
for i, t in enumerate(c["turns"]):
wav, sr = sf.read(DATA / c["id"] / f"{i:02d}.flac", dtype="float32")
s.turn(t["speaker"])
for k in range(0, len(wav), hop):
t0 = time.perf_counter(); s.push(wav[k:k + hop], sr); lat_push.append(time.perf_counter() - t0)
t0 = time.perf_counter(); ans, alert = s.ask(); lat_ask.append(time.perf_counter() - t0)
if alert and alert_turn is None:
alert_turn = i
gold_scam = c["true_from_turn"].get("scam") is not None and i >= int(c["true_from_turn"]["scam"])
right = (ans["scam"]["noul"] >= 0.5) == gold_scam; correct.append(right)
if prev_right and not right:
flips += 1
prev_right = right
# full re-encode of the same prefix
of = rt.predict_questions(state_parts(c, d["speakers"], upto=i), qs)
for k in qs:
stream.append(answer_probs(ans[k])); full.append(answer_probs(of["answers"][k]))
s.close()
res["calls"][c["id"]] = {"label": c["label"], "lang": c["lang"], "turns": len(c["turns"]), "decisive_turn": c["decisive_turn"], "alert_turn": alert_turn, "scam_prefix_acc": round(float(np.mean(correct)), 3), "flips_right_to_wrong": flips, "final": {k: (a.get("choice") or round(a["noul"], 3)) for k, a in ans.items()}}
res["streaming_vs_full"] = agreement(stream, full)
res["latency_ms"] = {"push_chunk_p50": round(float(np.percentile(lat_push, 50)) * 1000, 2), "push_chunk_p95": round(float(np.percentile(lat_push, 95)) * 1000, 2), "ask_p50": round(float(np.percentile(lat_ask, 50)) * 1000, 1), "ask_p95": round(float(np.percentile(lat_ask, 95)) * 1000, 1)}
if verbose:
print(json.dumps(res, ensure_ascii=False))
assert res["streaming_vs_full"]["argmax_agree"] >= 0.95, res["streaming_vs_full"]
for cid, r in res["calls"].items():
if r["label"] == "scam":
assert r["alert_turn"] is not None, f"no alert on scam call {cid}"
return res
if __name__ == "__main__":
ap = argparse.ArgumentParser(); ap.add_argument("--device"); ap.add_argument("--precision", default="bf16")
a = ap.parse_args(); run(a.device, a.precision)