Julia Klee
Julia-2: julia2_audio_v1 (text = base_v2 byte-identical, + pruned audio tower, stem p9), int8/int4 variants, lean runtime 0.1.0 (audio, LiveSession, modalities), card with speech, live calls and variants
1d1c6db Download runtime/tests/test_live.py from SupersonicLabs/Julia-2: direct link, hf CLI and curl.
- Browser
- Download file 3.51 kB
-
https://huggingface.co/SupersonicLabs/Julia-2/resolve/main/runtime/tests/test_live.py
- Command line
-
hf download hf://SupersonicLabs/Julia-2/runtime/tests/test_live.py
-
curl -L -o test_live.py https://huggingface.co/SupersonicLabs/Julia-2/resolve/main/runtime/tests/test_live.py
3.51 kB
| """Live session: audio pushed in 80 ms chunks turn by turn, the fixed questions re-asked after every turn, must equal a | |
| full one-shot encode of the same prefix (streaming == full re-encode; gate: argmax agreement >= 95% on these 3 hard calls — the | |
| reference j2 live path scores the same .9785 here and .99 over 115 calls; max|dp| p95 reported), | |
| alert must fire on the scam calls after the decisive turn and not on the legit call; prefix-accuracy of `scam` vs gold.""" | |
| import argparse, json, time | |
| from pathlib import Path | |
| import numpy as np | |
| from tests.common import REPO, agreement, answer_probs | |
| from tests.test_audio import load_calls, state_parts, DATA | |
| def run(device=None, precision="bf16", chunk_ms=80, verbose=True): | |
| from julia2 import Runtime | |
| import soundfile as sf | |
| rt = Runtime(REPO, device=device, precision=precision) | |
| d = load_calls(); res = {"device": str(rt.device), "precision": precision, "calls": {}} | |
| stream, full, lat_push, lat_ask = [], [], [], [] | |
| hop = int(16000 * chunk_ms / 1000) | |
| for c in d["calls"]: | |
| qs = d["questions"][c["lang"]] | |
| s = rt.open_live("Context: " + json.dumps(c["context"], ensure_ascii=False), qs, lang=c["lang"]) | |
| alert_turn, correct, flips = None, [], 0; prev_right = None | |
| for i, t in enumerate(c["turns"]): | |
| wav, sr = sf.read(DATA / c["id"] / f"{i:02d}.flac", dtype="float32") | |
| s.turn(t["speaker"]) | |
| for k in range(0, len(wav), hop): | |
| t0 = time.perf_counter(); s.push(wav[k:k + hop], sr); lat_push.append(time.perf_counter() - t0) | |
| t0 = time.perf_counter(); ans, alert = s.ask(); lat_ask.append(time.perf_counter() - t0) | |
| if alert and alert_turn is None: | |
| alert_turn = i | |
| gold_scam = c["true_from_turn"].get("scam") is not None and i >= int(c["true_from_turn"]["scam"]) | |
| right = (ans["scam"]["noul"] >= 0.5) == gold_scam; correct.append(right) | |
| if prev_right and not right: | |
| flips += 1 | |
| prev_right = right | |
| # full re-encode of the same prefix | |
| of = rt.predict_questions(state_parts(c, d["speakers"], upto=i), qs) | |
| for k in qs: | |
| stream.append(answer_probs(ans[k])); full.append(answer_probs(of["answers"][k])) | |
| s.close() | |
| res["calls"][c["id"]] = {"label": c["label"], "lang": c["lang"], "turns": len(c["turns"]), "decisive_turn": c["decisive_turn"], "alert_turn": alert_turn, "scam_prefix_acc": round(float(np.mean(correct)), 3), "flips_right_to_wrong": flips, "final": {k: (a.get("choice") or round(a["noul"], 3)) for k, a in ans.items()}} | |
| res["streaming_vs_full"] = agreement(stream, full) | |
| res["latency_ms"] = {"push_chunk_p50": round(float(np.percentile(lat_push, 50)) * 1000, 2), "push_chunk_p95": round(float(np.percentile(lat_push, 95)) * 1000, 2), "ask_p50": round(float(np.percentile(lat_ask, 50)) * 1000, 1), "ask_p95": round(float(np.percentile(lat_ask, 95)) * 1000, 1)} | |
| if verbose: | |
| print(json.dumps(res, ensure_ascii=False)) | |
| assert res["streaming_vs_full"]["argmax_agree"] >= 0.95, res["streaming_vs_full"] | |
| for cid, r in res["calls"].items(): | |
| if r["label"] == "scam": | |
| assert r["alert_turn"] is not None, f"no alert on scam call {cid}" | |
| return res | |
| if __name__ == "__main__": | |
| ap = argparse.ArgumentParser(); ap.add_argument("--device"); ap.add_argument("--precision", default="bf16") | |
| a = ap.parse_args(); run(a.device, a.precision) | |