Julia Klee
Julia-2: julia2_audio_v1 (text = base_v2 byte-identical, + pruned audio tower, stem p9), int8/int4 variants, lean runtime 0.1.0 (audio, LiveSession, modalities), card with speech, live calls and variants
1d1c6db Download runtime/tools/bench.py from SupersonicLabs/Julia-2: direct link, hf CLI and curl.
- Browser
- Download file 5.29 kB
-
https://huggingface.co/SupersonicLabs/Julia-2/resolve/main/runtime/tools/bench.py
- Command line
-
hf download hf://SupersonicLabs/Julia-2/runtime/tools/bench.py
-
curl -L -o bench.py https://huggingface.co/SupersonicLabs/Julia-2/resolve/main/runtime/tools/bench.py
5.29 kB
| """Latency / memory bench of the Julia-2 runtime: text (1 question, 225-token state), text shared-state (1 state x 6 questions), | |
| batch of 64 single questions, image (1.6 MP cap), audio one-shot (10 s clip), live hop (0.96 s audio + 6 questions), | |
| for a device and precision. Prints p50/p95 in ms and peak memory. | |
| python -m tools.bench [--device cuda|cpu] [--precision bf16|int8|int4] [--n 20]""" | |
| import argparse, json, os, sys, time | |
| from pathlib import Path | |
| import numpy as np, torch | |
| HERE = Path(__file__).resolve().parent | |
| sys.path.insert(0, str(HERE.parent)) | |
| REPO = Path(os.environ.get("JULIA2_REPO", HERE.parent.parent)) | |
| STATE = ("Customer Ana Ribeiro (account since 2019, premium tier) reports that her card was declined twice at a supermarket " | |
| "this morning although her balance is 2,340 EUR. The last three transactions were approved. She travelled abroad " | |
| "yesterday and did not notify the bank. Fraud rules: a card is soft-blocked after two declines in a new country " | |
| "unless travel was announced. She asks for an immediate unblock and a callback within one hour.") * 2 | |
| QS = {"action": {"type": "choice", "instructions": "What should the agent do first?", "criteria": {"unblock": "unblock the card after identity verification", "escalate": "escalate to the fraud team", "deny": "refuse and close", "callback": "schedule a callback only"}}, | |
| "fraud": {"type": "noul", "instructions": "Is this a likely fraud case?"}, "travel": {"type": "noul", "instructions": "Did the customer announce the travel?"}, | |
| "urgency": {"type": "score", "instructions": "How urgent is the request?", "criteria": ["not urgent", "normal", "urgent", "critical"]}, | |
| "tier": {"type": "choice", "instructions": "Which customer tier?", "criteria": {"basic": "basic", "premium": "premium", "unknown": "not stated"}}, | |
| "unblock_ok": {"type": "noul", "instructions": "Would unblocking the card now comply with the fraud rules?"}} | |
| def timed(fn, n, sync): | |
| ts = [] | |
| for _ in range(n): | |
| if sync: torch.cuda.synchronize() | |
| t0 = time.perf_counter(); fn() | |
| if sync: torch.cuda.synchronize() | |
| ts.append((time.perf_counter() - t0) * 1000) | |
| return {"p50": round(float(np.percentile(ts, 50)), 1), "p95": round(float(np.percentile(ts, 95)), 1)} | |
| def main(): | |
| ap = argparse.ArgumentParser(); ap.add_argument("--device"); ap.add_argument("--precision", default="bf16"); ap.add_argument("--n", type=int, default=20); ap.add_argument("--out") | |
| a = ap.parse_args() | |
| from julia2 import Runtime | |
| from tests.test_image import make_images | |
| t0 = time.time(); rt = Runtime(REPO, device=a.device, precision=a.precision); load_s = round(time.time() - t0, 1) | |
| sync = rt.device.type == "cuda"; n = a.n if sync else max(3, a.n // 5) | |
| if sync: torch.cuda.reset_peak_memory_stats() | |
| one = {"action": QS["action"]} | |
| res = {"device": str(rt.device), "precision": a.precision, "load_s": load_s, "state_tokens": len(rt.tok.encode(STATE)), "threads": torch.get_num_threads() if not sync else None} | |
| for _ in range(3): rt.predict_questions(STATE, one); rt.predict_questions(STATE, QS) | |
| res["text_1q_ms"] = timed(lambda: rt.predict_questions(STATE, one), n, sync) | |
| res["text_6q_shared_ms"] = timed(lambda: rt.predict_questions(STATE, QS, shared=True), n, sync) | |
| res["text_6q_separate_ms"] = timed(lambda: rt.predict_questions(STATE, QS, shared=False), n, sync) | |
| reqs = [{"state": STATE + f" Case {i}.", "questions": one} for i in range(64)] | |
| rt.predict_batch(reqs, shared=False) | |
| res["text_batch64_1q_ms_total"] = timed(lambda: rt.predict_batch(reqs, shared=False), max(2, n // 4), sync) | |
| if rt.vision is not None: | |
| from PIL import Image | |
| img = make_images()["red_square"].resize((1600, 1000)) | |
| q = {"color": QS["action"] | {"instructions": "What is the dominant colour?", "criteria": {"red": "red", "blue": "blue", "none": "none"}}} | |
| rt.predict_questions("", q, images=[img]); rt._img_cache.clear() | |
| def fresh(): | |
| rt._img_cache.clear(); rt.predict_questions("", q, images=[img]) | |
| res["image_1.6MP_1q_ms"] = timed(fresh, max(3, n // 2), sync) | |
| res["image_cached_1q_ms"] = timed(lambda: rt.predict_questions("", q, images=[img]), n, sync) | |
| if rt.audio is not None: | |
| rng = np.random.default_rng(0); wav = (rng.standard_normal(16000 * 10) * 0.05).astype(np.float32) | |
| rt.predict_questions("", one, audio=[(wav, 16000)]) | |
| res["audio_10s_1q_ms"] = timed(lambda: rt.predict_questions("", one, audio=[(wav, 16000)]), max(3, n // 2), sync) | |
| s = rt.open_live("Context: bench", QS, lang="en"); s.turn("caller"); hop = (rng.standard_normal(int(16000 * 0.96)) * 0.05).astype(np.float32) | |
| for _ in range(3): s.push(hop); s.ask() | |
| res["live_hop_push_0.96s_ms"] = timed(lambda: s.push(hop), n, sync) | |
| res["live_ask_6q_ms"] = timed(lambda: s.ask(), n, sync) | |
| if sync: | |
| res["peak_mem_gb"] = round(torch.cuda.max_memory_allocated() / 1e9, 2) | |
| else: | |
| import resource; res["peak_rss_gb"] = round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1e6, 2) | |
| print(json.dumps(res)) | |
| if a.out: | |
| Path(a.out).write_text(json.dumps(res, indent=1)) | |
| if __name__ == "__main__": | |
| main() | |