#!/usr/bin/env python # SPDX-License-Identifier: Apache-2.0 """Latency of the RDT2-VQ profile on one p150 (all prompt buckets resident, traced): per bucket the prefill trace and the decode-step trace (blocking replays), the RVQ decoder, and whole calls through the Python API (host pre-processing, the 28-token generation loop, the RVQ decode, the unnormalization). $ROOT/bin/devrun -t 1500 -- python -u code/scripts/vq_bench.py --iters 100 --json out.json [--dispatch worker] Environment: RDT2_VQ_RVQ (device | host), RDT2_VQ_DECODE_W (bf16 | bfp8), RDT2_VQ_LOGITS (fp32 | bf16). """ from __future__ import annotations import argparse import json import os import statistics import sys import time sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) def med(xs): return statistics.median(xs) def main(): ap = argparse.ArgumentParser() ap.add_argument("--iters", type=int, default=100) ap.add_argument("--calls", type=int, default=30) ap.add_argument("--dispatch", default=os.environ.get("RDT2_DISPATCH", "eth")) ap.add_argument("--json", default=None) args = ap.parse_args() import ttnn from tt_rdt2.api import load_sample from tt_rdt2.reference.config import DEFAULT_DIRS from tt_rdt2.vq.api import RDT2VQ wd = {} rvq = os.environ.get("RDT2_RVQ_DIR") or os.path.join(os.path.dirname(DEFAULT_DIRS["normalizer"]), "hf_RVQActionTokenizer_a6a614ae") for role, d in (("vlm", DEFAULT_DIRS["vlm"]), ("normalizer", DEFAULT_DIRS["normalizer"]), ("processor", DEFAULT_DIRS["processor"]), ("rvq", rvq)): if os.path.isdir(d): wd[role] = d t0 = time.time() model = RDT2VQ.from_pretrained(dispatch=args.dispatch, weights_dirs=wd, log_fn=print) boot = time.time() - t0 tt, dev = model._m, model.device res = {"dispatch": args.dispatch, "boot_s": boot, "grid": model.device_info.get("grid"), "decode_w": tt.decode_w, "rvq": tt.rvq_engine, "logits": str(tt.logits_dtype), "buckets": {}} s = load_sample() with model._lock: for L in tt.buckets: tt.check_allocator() pre, dec = [], [] for _ in range(args.iters): a = time.perf_counter() ttnn.execute_trace(dev, tt.traces[("prefill", L)], cq_id=0, blocking=True) pre.append((time.perf_counter() - a) * 1e3) for _ in range(args.iters): a = time.perf_counter() ttnn.execute_trace(dev, tt.traces["decode"], cq_id=0, blocking=True) dec.append((time.perf_counter() - a) * 1e3) res["buckets"][L] = {"prefill_ms": med(pre), "decode_step_ms": med(dec)} print(f"bucket {L}: prefill {med(pre):.2f} ms, decode step {med(dec):.2f} ms", flush=True) if tt.rvq_engine == "device": import torch rv = [] tok = torch.zeros(1, 27, dtype=torch.long) for _ in range(args.iters): a = time.perf_counter() tt.rvq.decode(tok) rv.append((time.perf_counter() - a) * 1e3) res["rvq_decode_ms"] = med(rv) calls, parts = [], [] for _ in range(args.calls): a = time.perf_counter() out = model(s["images"], s["instruction"]) calls.append((time.perf_counter() - a) * 1e3) parts.append(out.timing_ms) res["call_ms"] = {"median": med(calls), "min": min(calls), "max": max(calls), "n": len(calls)} res["call_parts_median_ms"] = {k: med([p[k] for p in parts]) for k in parts[0]} res["tokens"] = len(out.generated) res["slow_path"] = dict(tt.stats) res["host_load"] = os.getloadavg() print(json.dumps(res, indent=1), flush=True) if args.json: with open(args.json, "w") as f: json.dump(res, f, indent=1) model.close() if __name__ == "__main__": main()