"""Latency / memory bench of the Julia-2 runtime: text (1 question, 225-token state), text shared-state (1 state x 6 questions), batch of 64 single questions, image (1.6 MP cap), audio one-shot (10 s clip), live hop (0.96 s audio + 6 questions), for a device and precision. Prints p50/p95 in ms and peak memory. python -m tools.bench [--device cuda|cpu] [--precision bf16|int8|int4] [--n 20]""" import argparse, json, os, sys, time from pathlib import Path import numpy as np, torch HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE.parent)) REPO = Path(os.environ.get("JULIA2_REPO", HERE.parent.parent)) STATE = ("Customer Ana Ribeiro (account since 2019, premium tier) reports that her card was declined twice at a supermarket " "this morning although her balance is 2,340 EUR. The last three transactions were approved. She travelled abroad " "yesterday and did not notify the bank. Fraud rules: a card is soft-blocked after two declines in a new country " "unless travel was announced. She asks for an immediate unblock and a callback within one hour.") * 2 QS = {"action": {"type": "choice", "instructions": "What should the agent do first?", "criteria": {"unblock": "unblock the card after identity verification", "escalate": "escalate to the fraud team", "deny": "refuse and close", "callback": "schedule a callback only"}}, "fraud": {"type": "noul", "instructions": "Is this a likely fraud case?"}, "travel": {"type": "noul", "instructions": "Did the customer announce the travel?"}, "urgency": {"type": "score", "instructions": "How urgent is the request?", "criteria": ["not urgent", "normal", "urgent", "critical"]}, "tier": {"type": "choice", "instructions": "Which customer tier?", "criteria": {"basic": "basic", "premium": "premium", "unknown": "not stated"}}, "unblock_ok": {"type": "noul", "instructions": "Would unblocking the card now comply with the fraud rules?"}} def timed(fn, n, sync): ts = [] for _ in range(n): if sync: torch.cuda.synchronize() t0 = time.perf_counter(); fn() if sync: torch.cuda.synchronize() ts.append((time.perf_counter() - t0) * 1000) return {"p50": round(float(np.percentile(ts, 50)), 1), "p95": round(float(np.percentile(ts, 95)), 1)} def main(): ap = argparse.ArgumentParser(); ap.add_argument("--device"); ap.add_argument("--precision", default="bf16"); ap.add_argument("--n", type=int, default=20); ap.add_argument("--out") ap.add_argument("--text-only", action="store_true", help="load only the decision trunk (no vision or audio tower)") a = ap.parse_args() from julia2 import Runtime from tests.test_image import make_images t0 = time.time(); rt = Runtime(REPO, device=a.device, precision=a.precision, modalities=("text",) if a.text_only else ("text", "image", "audio")); load_s = round(time.time() - t0, 1) sync = rt.device.type == "cuda"; n = a.n if sync else max(3, a.n // 5) if sync: torch.cuda.reset_peak_memory_stats() one = {"action": QS["action"]} res = {"device": str(rt.device), "precision": a.precision, "load_s": load_s, "state_tokens": len(rt.tok.encode(STATE)), "threads": torch.get_num_threads() if not sync else None} for _ in range(3): rt.predict_questions(STATE, one); rt.predict_questions(STATE, QS) res["text_1q_ms"] = timed(lambda: rt.predict_questions(STATE, one), n, sync) res["text_6q_shared_ms"] = timed(lambda: rt.predict_questions(STATE, QS, shared=True), n, sync) res["text_6q_separate_ms"] = timed(lambda: rt.predict_questions(STATE, QS, shared=False), n, sync) reqs = [{"state": STATE + f" Case {i}.", "questions": one} for i in range(64)] rt.predict_batch(reqs, shared=False) res["text_batch64_1q_ms_total"] = timed(lambda: rt.predict_batch(reqs, shared=False), max(2, n // 4), sync) if rt.vision is not None: from PIL import Image img = make_images()["red_square"].resize((1600, 1000)) q = {"color": QS["action"] | {"instructions": "What is the dominant colour?", "criteria": {"red": "red", "blue": "blue", "none": "none"}}} rt.predict_questions("", q, images=[img]); rt._img_cache.clear() def fresh(): rt._img_cache.clear(); rt.predict_questions("", q, images=[img]) res["image_1.6MP_1q_ms"] = timed(fresh, max(3, n // 2), sync) res["image_cached_1q_ms"] = timed(lambda: rt.predict_questions("", q, images=[img]), n, sync) if rt.audio is not None: rng = np.random.default_rng(0); wav = (rng.standard_normal(16000 * 10) * 0.05).astype(np.float32) rt.predict_questions("", one, audio=[(wav, 16000)]) res["audio_10s_1q_ms"] = timed(lambda: rt.predict_questions("", one, audio=[(wav, 16000)]), max(3, n // 2), sync) s = rt.open_live("Context: bench", QS, lang="en"); s.turn("caller"); hop = (rng.standard_normal(int(16000 * 0.96)) * 0.05).astype(np.float32) for _ in range(3): s.push(hop); s.ask() res["live_hop_push_0.96s_ms"] = timed(lambda: s.push(hop), n, sync) res["live_ask_6q_ms"] = timed(lambda: s.ask(), n, sync) if sync: res["peak_mem_gb"] = round(torch.cuda.max_memory_allocated() / 1e9, 2) res["reserved_gb"] = round(torch.cuda.memory_reserved() / 1e9, 2) else: import resource; res["peak_rss_gb"] = round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (1e9 if sys.platform == "darwin" else 1e6), 2) print(json.dumps(res)) if a.out: Path(a.out).write_text(json.dumps(res, indent=1)) if __name__ == "__main__": main()