# SPDX-License-Identifier: Apache-2.0 """Latency of RDT2-FM (engine per program: RDT2_ENGINE, e.g. expert=mk): trace replay per call, per stage, per expert step, host pre / post, H2D / D2H. python -u code/scripts/bench.py --iters 40 --json bench.json # from the repo root (one chip; ETH) Weights: the ``RDT2_*_DIR`` directories, else the pinned snapshots in the Hugging Face cache (``tt_rdt2.api .resolve_weights``: the same resolution as the server). Also measures the Python API call (``RDT2FM.__call__``: request validation + the call; ``api_call_ms``) on the same resident traces. Input: the shipped Apache-2.0 sample (code/tt_rdt2/samples/ur5e_bimanual_example_0.npz) = golden real_ur5e__s0. All ``--buckets`` are resident (one trace each); the sample's own bucket (416) gives the headline numbers and the same request is also forced into every resident bucket (``trace_replay_ms_by_bucket``, ``stages_by_bucket``). Each number is the median of ``--iters`` repetitions after 5 warm-up replays; device times are host wall-clock around ``execute_trace(blocking=False)`` + ``synchronize_device`` (one call at a time, batch 1). Every replay first runs the memory-contract check (``check_persistent``: persistent addresses + allocator totals, about 0.6 ms of host time), so it is inside the trace-replay, stage and e2e numbers. """ from __future__ import annotations import argparse import json import os import statistics import sys import time import numpy as np import torch sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) from tt_rdt2.tt.model import RDT2FMTT, open_rdt2_device # noqa: E402 SAMPLE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "tt_rdt2", "samples", "ur5e_bimanual_example_0.npz") def med(xs): return float(statistics.median(xs)) def main(): ap = argparse.ArgumentParser() ap.add_argument("--precision", default=None) ap.add_argument("--buckets", default="416,448,512", help="resident prompt buckets (one trace each)") ap.add_argument("--dispatch", default="eth") ap.add_argument("--iters", type=int, default=60) ap.add_argument("--json", default=None) ap.add_argument("--no-stages", action="store_true") a = ap.parse_args() import ttnn s = np.load(SAMPLE, allow_pickle=False) images = [s["left_stereo"], s["right_stereo"]] if "left_stereo" in s.files else [s["left"], s["right"]] instr = str(s["instruction"]) if "instruction" in s.files else "Pour water from the plastic bottle into the cup" state = s["state"] if "state" in s.files else np.zeros(20, np.float32) noise = torch.randn((1, 24, 20), generator=torch.Generator().manual_seed(0)) dev = open_rdt2_device(a.dispatch) g = dev.compute_with_storage_grid_size() buckets = tuple(int(b) for b in a.buckets.split(",")) out = {"grid": f"{g.x}x{g.y}", "dispatch": a.dispatch, "buckets": list(buckets), "iters": a.iters, "sample": os.path.basename(SAMPLE), "torch_threads": torch.get_num_threads()} try: from tt_rdt2.api import RDT2FM, resolve_weights from tt_rdt2.reference.config import DEFAULT_DIRS w = resolve_weights(weights_dirs={r: d for r, d in DEFAULT_DIRS.items() if r != "golden" and os.path.isdir(d)}, log_fn=lambda x: None) m = RDT2FMTT(dev, buckets=buckets, precision=a.precision, vlm_dir=w["vlm"]["path"], expert_dir=w["expert"]["path"], normalizer_dir=w["normalizer"]["path"], processor_dir=w["processor"]["path"]) out["precision"] = m.prec.as_dict() out["load_s"] = m.timings["load_s"] # host pre hp = [] for _ in range(a.iters): t0 = time.perf_counter() host = m.host_inputs(images, instr, state, noise) hp.append((time.perf_counter() - t0) * 1e3) out["host_pre_ms"] = med(hp) L0 = host["bucket"] out["bucket"] = L0 out["n_tokens"] = int(host["n_tokens"]) t0 = time.perf_counter() m.capture() out["capture_s_all_buckets"] = time.perf_counter() - t0 # H2D (5 input buffers) h2d = [] for _ in range(a.iters): ttnn.synchronize_device(dev) t0 = time.perf_counter() m.write_inputs(host) ttnn.synchronize_device(dev) h2d.append((time.perf_counter() - t0) * 1e3) out["h2d_ms"] = med(h2d) def time_replay(L): for _ in range(5): m.replay(L, blocking=True) rep = [] for _ in range(a.iters): t0 = time.perf_counter() m.replay(L, blocking=False) ttnn.synchronize_device(dev) rep.append((time.perf_counter() - t0) * 1e3) return rep by_bucket = {} for L in m.buckets: # the same request forced into every resident bucket m.write_inputs(m.host_inputs(images, instr, state, noise, bucket=L)) by_bucket[str(L)] = med(time_replay(L)) out["trace_replay_ms_by_bucket"] = by_bucket m.write_inputs(host) rep = time_replay(L0) out["trace_replay_ms"] = med(rep) out["trace_replay_ms_p10_p90"] = [float(np.percentile(rep, 10)), float(np.percentile(rep, 90))] d2h = [] for _ in range(a.iters): t0 = time.perf_counter() ns = m.read_action_normalized(L0) d2h.append((time.perf_counter() - t0) * 1e3) out["d2h_ms"] = med(d2h) from tt_rdt2.reference.expert import unnormalize_action hpost = [] for _ in range(a.iters): t0 = time.perf_counter() unnormalize_action(ns, m.norm) hpost.append((time.perf_counter() - t0) * 1e3) out["host_post_ms"] = med(hpost) e2e = [] for _ in range(a.iters): t0 = time.perf_counter() r = m(images, instr, state, noise) e2e.append((time.perf_counter() - t0) * 1e3) out["e2e_call_ms"] = med(e2e) out["e2e_call_ms_p10_p90"] = [float(np.percentile(e2e, 10)), float(np.percentile(e2e, 90))] out["action_first_row"] = r["action"][0].tolist() api_model = RDT2FM(m, dev, weights=w, owns_device=False, device_id=None, precision=a.precision or "baseline", log_fn=lambda x: None) apis = [] for _ in range(a.iters): t0 = time.perf_counter() api_model(images, instr, state=state, seed=0) apis.append((time.perf_counter() - t0) * 1e3) out["api_call_ms"] = med(apis) out["api_call_ms_p10_p90"] = [float(np.percentile(apis, 10)), float(np.percentile(apis, 90))] if not a.no_stages: ns_full = m.read_action_normalized(L0) out["stages_by_bucket"] = {} for L in m.buckets: h = m.host_inputs(images, instr, state, noise, bucket=L) tids = m.capture_stages(L) m.write_inputs(h) for name, tid in tids.items(): ttnn.execute_trace(dev, tid, cq_id=0, blocking=True) st = m.time_stages(a.iters) if "expert" in st: # megakernel: the whole action head is one stage (prologue matmul + program) r = dict(stage_ms=st, vision_ms=st["vision"], prefix_ms=st["prefix"], expert_total_ms=st["expert"], stage_sum_ms=sum(st.values())) else: steps = [st[f"step{k}"] for k in range(m.num_steps)] r = dict(stage_ms=st, vision_ms=st["vision"], prefix_ms=st["prefix"], expert_prologue_ms=st["prologue"], expert_per_step_ms=float(np.mean(steps)), expert_total_ms=st["prologue"] + sum(steps), stage_sum_ms=sum(st.values())) if L == L0: r["stage_chain_bitexact_vs_full"] = bool(torch.equal(m.read_stage_action(), ns_full)) out.update(r) out["stages_by_bucket"][str(L)] = r m.release_stage_traces() print(json.dumps(out, indent=1), flush=True) finally: if a.json: with open(a.json, "w") as f: json.dump(out, f, indent=1) from tt_rdt2.ttaw.device import close_device close_device(dev) if __name__ == "__main__": main()