File size: 3,972 Bytes
c869f1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 | #!/usr/bin/env python
# SPDX-License-Identifier: Apache-2.0
"""Latency of the RDT2-VQ profile on one p150 (all prompt buckets resident, traced): per bucket the prefill trace and
the decode-step trace (blocking replays), the RVQ decoder, and whole calls through the Python API (host pre-processing,
the 28-token generation loop, the RVQ decode, the unnormalization).
$ROOT/bin/devrun -t 1500 -- python -u code/scripts/vq_bench.py --iters 100 --json out.json [--dispatch worker]
Environment: RDT2_VQ_RVQ (device | host), RDT2_VQ_DECODE_W (bf16 | bfp8), RDT2_VQ_LOGITS (fp32 | bf16).
"""
from __future__ import annotations
import argparse
import json
import os
import statistics
import sys
import time
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
def med(xs):
return statistics.median(xs)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--iters", type=int, default=100)
ap.add_argument("--calls", type=int, default=30)
ap.add_argument("--dispatch", default=os.environ.get("RDT2_DISPATCH", "eth"))
ap.add_argument("--json", default=None)
args = ap.parse_args()
import ttnn
from tt_rdt2.api import load_sample
from tt_rdt2.reference.config import DEFAULT_DIRS
from tt_rdt2.vq.api import RDT2VQ
wd = {}
rvq = os.environ.get("RDT2_RVQ_DIR") or os.path.join(os.path.dirname(DEFAULT_DIRS["normalizer"]),
"hf_RVQActionTokenizer_a6a614ae")
for role, d in (("vlm", DEFAULT_DIRS["vlm"]), ("normalizer", DEFAULT_DIRS["normalizer"]),
("processor", DEFAULT_DIRS["processor"]), ("rvq", rvq)):
if os.path.isdir(d):
wd[role] = d
t0 = time.time()
model = RDT2VQ.from_pretrained(dispatch=args.dispatch, weights_dirs=wd, log_fn=print)
boot = time.time() - t0
tt, dev = model._m, model.device
res = {"dispatch": args.dispatch, "boot_s": boot, "grid": model.device_info.get("grid"),
"decode_w": tt.decode_w, "rvq": tt.rvq_engine, "logits": str(tt.logits_dtype), "buckets": {}}
s = load_sample()
with model._lock:
for L in tt.buckets:
tt.check_allocator()
pre, dec = [], []
for _ in range(args.iters):
a = time.perf_counter()
ttnn.execute_trace(dev, tt.traces[("prefill", L)], cq_id=0, blocking=True)
pre.append((time.perf_counter() - a) * 1e3)
for _ in range(args.iters):
a = time.perf_counter()
ttnn.execute_trace(dev, tt.traces["decode"], cq_id=0, blocking=True)
dec.append((time.perf_counter() - a) * 1e3)
res["buckets"][L] = {"prefill_ms": med(pre), "decode_step_ms": med(dec)}
print(f"bucket {L}: prefill {med(pre):.2f} ms, decode step {med(dec):.2f} ms", flush=True)
if tt.rvq_engine == "device":
import torch
rv = []
tok = torch.zeros(1, 27, dtype=torch.long)
for _ in range(args.iters):
a = time.perf_counter()
tt.rvq.decode(tok)
rv.append((time.perf_counter() - a) * 1e3)
res["rvq_decode_ms"] = med(rv)
calls, parts = [], []
for _ in range(args.calls):
a = time.perf_counter()
out = model(s["images"], s["instruction"])
calls.append((time.perf_counter() - a) * 1e3)
parts.append(out.timing_ms)
res["call_ms"] = {"median": med(calls), "min": min(calls), "max": max(calls), "n": len(calls)}
res["call_parts_median_ms"] = {k: med([p[k] for p in parts]) for k in parts[0]}
res["tokens"] = len(out.generated)
res["slow_path"] = dict(tt.stats)
res["host_load"] = os.getloadavg()
print(json.dumps(res, indent=1), flush=True)
if args.json:
with open(args.json, "w") as f:
json.dump(res, f, indent=1)
model.close()
if __name__ == "__main__":
main()
|