vianahnz's picture
Julia 2 Flash
b61b4c4 verified
Raw History Blame Contribute Delete
5.59 kB
"""Latency / memory bench of the Julia-2 runtime: text (1 question, 225-token state), text shared-state (1 state x 6 questions),
batch of 64 single questions, image (1.6 MP cap), audio one-shot (10 s clip), live hop (0.96 s audio + 6 questions),
for a device and precision. Prints p50/p95 in ms and peak memory.
python -m tools.bench [--device cuda|cpu] [--precision bf16|int8|int4] [--n 20]"""
import argparse, json, os, sys, time
from pathlib import Path
import numpy as np, torch
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE.parent))
REPO = Path(os.environ.get("JULIA2_REPO", HERE.parent.parent))
STATE = ("Customer Ana Ribeiro (account since 2019, premium tier) reports that her card was declined twice at a supermarket "
"this morning although her balance is 2,340 EUR. The last three transactions were approved. She travelled abroad "
"yesterday and did not notify the bank. Fraud rules: a card is soft-blocked after two declines in a new country "
"unless travel was announced. She asks for an immediate unblock and a callback within one hour.") * 2
QS = {"action": {"type": "choice", "instructions": "What should the agent do first?", "criteria": {"unblock": "unblock the card after identity verification", "escalate": "escalate to the fraud team", "deny": "refuse and close", "callback": "schedule a callback only"}},
"fraud": {"type": "noul", "instructions": "Is this a likely fraud case?"}, "travel": {"type": "noul", "instructions": "Did the customer announce the travel?"},
"urgency": {"type": "score", "instructions": "How urgent is the request?", "criteria": ["not urgent", "normal", "urgent", "critical"]},
"tier": {"type": "choice", "instructions": "Which customer tier?", "criteria": {"basic": "basic", "premium": "premium", "unknown": "not stated"}},
"unblock_ok": {"type": "noul", "instructions": "Would unblocking the card now comply with the fraud rules?"}}
def timed(fn, n, sync):
ts = []
for _ in range(n):
if sync: torch.cuda.synchronize()
t0 = time.perf_counter(); fn()
if sync: torch.cuda.synchronize()
ts.append((time.perf_counter() - t0) * 1000)
return {"p50": round(float(np.percentile(ts, 50)), 1), "p95": round(float(np.percentile(ts, 95)), 1)}
def main():
ap = argparse.ArgumentParser(); ap.add_argument("--device"); ap.add_argument("--precision", default="bf16"); ap.add_argument("--n", type=int, default=20); ap.add_argument("--out")
ap.add_argument("--text-only", action="store_true", help="load only the decision trunk (no vision or audio tower)")
a = ap.parse_args()
from julia2 import Runtime
from tests.test_image import make_images
t0 = time.time(); rt = Runtime(REPO, device=a.device, precision=a.precision, modalities=("text",) if a.text_only else ("text", "image", "audio")); load_s = round(time.time() - t0, 1)
sync = rt.device.type == "cuda"; n = a.n if sync else max(3, a.n // 5)
if sync: torch.cuda.reset_peak_memory_stats()
one = {"action": QS["action"]}
res = {"device": str(rt.device), "precision": a.precision, "load_s": load_s, "state_tokens": len(rt.tok.encode(STATE)), "threads": torch.get_num_threads() if not sync else None}
for _ in range(3): rt.predict_questions(STATE, one); rt.predict_questions(STATE, QS)
res["text_1q_ms"] = timed(lambda: rt.predict_questions(STATE, one), n, sync)
res["text_6q_shared_ms"] = timed(lambda: rt.predict_questions(STATE, QS, shared=True), n, sync)
res["text_6q_separate_ms"] = timed(lambda: rt.predict_questions(STATE, QS, shared=False), n, sync)
reqs = [{"state": STATE + f" Case {i}.", "questions": one} for i in range(64)]
rt.predict_batch(reqs, shared=False)
res["text_batch64_1q_ms_total"] = timed(lambda: rt.predict_batch(reqs, shared=False), max(2, n // 4), sync)
if rt.vision is not None:
from PIL import Image
img = make_images()["red_square"].resize((1600, 1000))
q = {"color": QS["action"] | {"instructions": "What is the dominant colour?", "criteria": {"red": "red", "blue": "blue", "none": "none"}}}
rt.predict_questions("", q, images=[img]); rt._img_cache.clear()
def fresh():
rt._img_cache.clear(); rt.predict_questions("", q, images=[img])
res["image_1.6MP_1q_ms"] = timed(fresh, max(3, n // 2), sync)
res["image_cached_1q_ms"] = timed(lambda: rt.predict_questions("", q, images=[img]), n, sync)
if rt.audio is not None:
rng = np.random.default_rng(0); wav = (rng.standard_normal(16000 * 10) * 0.05).astype(np.float32)
rt.predict_questions("", one, audio=[(wav, 16000)])
res["audio_10s_1q_ms"] = timed(lambda: rt.predict_questions("", one, audio=[(wav, 16000)]), max(3, n // 2), sync)
s = rt.open_live("Context: bench", QS, lang="en"); s.turn("caller"); hop = (rng.standard_normal(int(16000 * 0.96)) * 0.05).astype(np.float32)
for _ in range(3): s.push(hop); s.ask()
res["live_hop_push_0.96s_ms"] = timed(lambda: s.push(hop), n, sync)
res["live_ask_6q_ms"] = timed(lambda: s.ask(), n, sync)
if sync:
res["peak_mem_gb"] = round(torch.cuda.max_memory_allocated() / 1e9, 2)
res["reserved_gb"] = round(torch.cuda.memory_reserved() / 1e9, 2)
else:
import resource; res["peak_rss_gb"] = round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (1e9 if sys.platform == "darwin" else 1e6), 2)
print(json.dumps(res))
if a.out:
Path(a.out).write_text(json.dumps(res, indent=1))
if __name__ == "__main__":
main()