Download code/scripts/vq_bench.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 3.97 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/scripts/vq_bench.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/scripts/vq_bench.py
-
curl -L -o vq_bench.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/scripts/vq_bench.py
3.97 kB
| #!/usr/bin/env python | |
| # SPDX-License-Identifier: Apache-2.0 | |
| """Latency of the RDT2-VQ profile on one p150 (all prompt buckets resident, traced): per bucket the prefill trace and | |
| the decode-step trace (blocking replays), the RVQ decoder, and whole calls through the Python API (host pre-processing, | |
| the 28-token generation loop, the RVQ decode, the unnormalization). | |
| $ROOT/bin/devrun -t 1500 -- python -u code/scripts/vq_bench.py --iters 100 --json out.json [--dispatch worker] | |
| Environment: RDT2_VQ_RVQ (device | host), RDT2_VQ_DECODE_W (bf16 | bfp8), RDT2_VQ_LOGITS (fp32 | bf16). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import statistics | |
| import sys | |
| import time | |
| sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) | |
| def med(xs): | |
| return statistics.median(xs) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--iters", type=int, default=100) | |
| ap.add_argument("--calls", type=int, default=30) | |
| ap.add_argument("--dispatch", default=os.environ.get("RDT2_DISPATCH", "eth")) | |
| ap.add_argument("--json", default=None) | |
| args = ap.parse_args() | |
| import ttnn | |
| from tt_rdt2.api import load_sample | |
| from tt_rdt2.reference.config import DEFAULT_DIRS | |
| from tt_rdt2.vq.api import RDT2VQ | |
| wd = {} | |
| rvq = os.environ.get("RDT2_RVQ_DIR") or os.path.join(os.path.dirname(DEFAULT_DIRS["normalizer"]), | |
| "hf_RVQActionTokenizer_a6a614ae") | |
| for role, d in (("vlm", DEFAULT_DIRS["vlm"]), ("normalizer", DEFAULT_DIRS["normalizer"]), | |
| ("processor", DEFAULT_DIRS["processor"]), ("rvq", rvq)): | |
| if os.path.isdir(d): | |
| wd[role] = d | |
| t0 = time.time() | |
| model = RDT2VQ.from_pretrained(dispatch=args.dispatch, weights_dirs=wd, log_fn=print) | |
| boot = time.time() - t0 | |
| tt, dev = model._m, model.device | |
| res = {"dispatch": args.dispatch, "boot_s": boot, "grid": model.device_info.get("grid"), | |
| "decode_w": tt.decode_w, "rvq": tt.rvq_engine, "logits": str(tt.logits_dtype), "buckets": {}} | |
| s = load_sample() | |
| with model._lock: | |
| for L in tt.buckets: | |
| tt.check_allocator() | |
| pre, dec = [], [] | |
| for _ in range(args.iters): | |
| a = time.perf_counter() | |
| ttnn.execute_trace(dev, tt.traces[("prefill", L)], cq_id=0, blocking=True) | |
| pre.append((time.perf_counter() - a) * 1e3) | |
| for _ in range(args.iters): | |
| a = time.perf_counter() | |
| ttnn.execute_trace(dev, tt.traces["decode"], cq_id=0, blocking=True) | |
| dec.append((time.perf_counter() - a) * 1e3) | |
| res["buckets"][L] = {"prefill_ms": med(pre), "decode_step_ms": med(dec)} | |
| print(f"bucket {L}: prefill {med(pre):.2f} ms, decode step {med(dec):.2f} ms", flush=True) | |
| if tt.rvq_engine == "device": | |
| import torch | |
| rv = [] | |
| tok = torch.zeros(1, 27, dtype=torch.long) | |
| for _ in range(args.iters): | |
| a = time.perf_counter() | |
| tt.rvq.decode(tok) | |
| rv.append((time.perf_counter() - a) * 1e3) | |
| res["rvq_decode_ms"] = med(rv) | |
| calls, parts = [], [] | |
| for _ in range(args.calls): | |
| a = time.perf_counter() | |
| out = model(s["images"], s["instruction"]) | |
| calls.append((time.perf_counter() - a) * 1e3) | |
| parts.append(out.timing_ms) | |
| res["call_ms"] = {"median": med(calls), "min": min(calls), "max": max(calls), "n": len(calls)} | |
| res["call_parts_median_ms"] = {k: med([p[k] for p in parts]) for k in parts[0]} | |
| res["tokens"] = len(out.generated) | |
| res["slow_path"] = dict(tt.stats) | |
| res["host_load"] = os.getloadavg() | |
| print(json.dumps(res, indent=1), flush=True) | |
| if args.json: | |
| with open(args.json, "w") as f: | |
| json.dump(res, f, indent=1) | |
| model.close() | |
| if __name__ == "__main__": | |
| main() | |