Download code/scripts/bench.py from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 8.49 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/scripts/bench.py
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/scripts/bench.py
-
curl -L -o bench.py https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/scripts/bench.py
8.49 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """Latency of RDT2-FM (engine per program: RDT2_ENGINE, e.g. expert=mk): trace replay per call, per stage, per expert step, host pre / post, H2D / D2H. | |
| python -u code/scripts/bench.py --iters 40 --json bench.json # from the repo root (one chip; ETH) | |
| Weights: the ``RDT2_*_DIR`` directories, else the pinned snapshots in the Hugging Face cache (``tt_rdt2.api | |
| .resolve_weights``: the same resolution as the server). Also measures the Python API call (``RDT2FM.__call__``: request | |
| validation + the call; ``api_call_ms``) on the same resident traces. | |
| Input: the shipped Apache-2.0 sample (code/tt_rdt2/samples/ur5e_bimanual_example_0.npz) = golden real_ur5e__s0. | |
| All ``--buckets`` are resident (one trace each); the sample's own bucket (416) gives the headline numbers and the same | |
| request is also forced into every resident bucket (``trace_replay_ms_by_bucket``, ``stages_by_bucket``). | |
| Each number is the median of ``--iters`` repetitions after 5 warm-up replays; device times are host wall-clock around | |
| ``execute_trace(blocking=False)`` + ``synchronize_device`` (one call at a time, batch 1). Every replay first runs the | |
| memory-contract check (``check_persistent``: persistent addresses + allocator totals, about 0.6 ms of host time), so it | |
| is inside the trace-replay, stage and e2e numbers. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import statistics | |
| import sys | |
| import time | |
| import numpy as np | |
| import torch | |
| sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) | |
| from tt_rdt2.tt.model import RDT2FMTT, open_rdt2_device # noqa: E402 | |
| SAMPLE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "tt_rdt2", "samples", | |
| "ur5e_bimanual_example_0.npz") | |
| def med(xs): | |
| return float(statistics.median(xs)) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--precision", default=None) | |
| ap.add_argument("--buckets", default="416,448,512", help="resident prompt buckets (one trace each)") | |
| ap.add_argument("--dispatch", default="eth") | |
| ap.add_argument("--iters", type=int, default=60) | |
| ap.add_argument("--json", default=None) | |
| ap.add_argument("--no-stages", action="store_true") | |
| a = ap.parse_args() | |
| import ttnn | |
| s = np.load(SAMPLE, allow_pickle=False) | |
| images = [s["left_stereo"], s["right_stereo"]] if "left_stereo" in s.files else [s["left"], s["right"]] | |
| instr = str(s["instruction"]) if "instruction" in s.files else "Pour water from the plastic bottle into the cup" | |
| state = s["state"] if "state" in s.files else np.zeros(20, np.float32) | |
| noise = torch.randn((1, 24, 20), generator=torch.Generator().manual_seed(0)) | |
| dev = open_rdt2_device(a.dispatch) | |
| g = dev.compute_with_storage_grid_size() | |
| buckets = tuple(int(b) for b in a.buckets.split(",")) | |
| out = {"grid": f"{g.x}x{g.y}", "dispatch": a.dispatch, "buckets": list(buckets), "iters": a.iters, | |
| "sample": os.path.basename(SAMPLE), "torch_threads": torch.get_num_threads()} | |
| try: | |
| from tt_rdt2.api import RDT2FM, resolve_weights | |
| from tt_rdt2.reference.config import DEFAULT_DIRS | |
| w = resolve_weights(weights_dirs={r: d for r, d in DEFAULT_DIRS.items() if r != "golden" and os.path.isdir(d)}, | |
| log_fn=lambda x: None) | |
| m = RDT2FMTT(dev, buckets=buckets, precision=a.precision, vlm_dir=w["vlm"]["path"], | |
| expert_dir=w["expert"]["path"], normalizer_dir=w["normalizer"]["path"], | |
| processor_dir=w["processor"]["path"]) | |
| out["precision"] = m.prec.as_dict() | |
| out["load_s"] = m.timings["load_s"] | |
| # host pre | |
| hp = [] | |
| for _ in range(a.iters): | |
| t0 = time.perf_counter() | |
| host = m.host_inputs(images, instr, state, noise) | |
| hp.append((time.perf_counter() - t0) * 1e3) | |
| out["host_pre_ms"] = med(hp) | |
| L0 = host["bucket"] | |
| out["bucket"] = L0 | |
| out["n_tokens"] = int(host["n_tokens"]) | |
| t0 = time.perf_counter() | |
| m.capture() | |
| out["capture_s_all_buckets"] = time.perf_counter() - t0 | |
| # H2D (5 input buffers) | |
| h2d = [] | |
| for _ in range(a.iters): | |
| ttnn.synchronize_device(dev) | |
| t0 = time.perf_counter() | |
| m.write_inputs(host) | |
| ttnn.synchronize_device(dev) | |
| h2d.append((time.perf_counter() - t0) * 1e3) | |
| out["h2d_ms"] = med(h2d) | |
| def time_replay(L): | |
| for _ in range(5): | |
| m.replay(L, blocking=True) | |
| rep = [] | |
| for _ in range(a.iters): | |
| t0 = time.perf_counter() | |
| m.replay(L, blocking=False) | |
| ttnn.synchronize_device(dev) | |
| rep.append((time.perf_counter() - t0) * 1e3) | |
| return rep | |
| by_bucket = {} | |
| for L in m.buckets: # the same request forced into every resident bucket | |
| m.write_inputs(m.host_inputs(images, instr, state, noise, bucket=L)) | |
| by_bucket[str(L)] = med(time_replay(L)) | |
| out["trace_replay_ms_by_bucket"] = by_bucket | |
| m.write_inputs(host) | |
| rep = time_replay(L0) | |
| out["trace_replay_ms"] = med(rep) | |
| out["trace_replay_ms_p10_p90"] = [float(np.percentile(rep, 10)), float(np.percentile(rep, 90))] | |
| d2h = [] | |
| for _ in range(a.iters): | |
| t0 = time.perf_counter() | |
| ns = m.read_action_normalized(L0) | |
| d2h.append((time.perf_counter() - t0) * 1e3) | |
| out["d2h_ms"] = med(d2h) | |
| from tt_rdt2.reference.expert import unnormalize_action | |
| hpost = [] | |
| for _ in range(a.iters): | |
| t0 = time.perf_counter() | |
| unnormalize_action(ns, m.norm) | |
| hpost.append((time.perf_counter() - t0) * 1e3) | |
| out["host_post_ms"] = med(hpost) | |
| e2e = [] | |
| for _ in range(a.iters): | |
| t0 = time.perf_counter() | |
| r = m(images, instr, state, noise) | |
| e2e.append((time.perf_counter() - t0) * 1e3) | |
| out["e2e_call_ms"] = med(e2e) | |
| out["e2e_call_ms_p10_p90"] = [float(np.percentile(e2e, 10)), float(np.percentile(e2e, 90))] | |
| out["action_first_row"] = r["action"][0].tolist() | |
| api_model = RDT2FM(m, dev, weights=w, owns_device=False, device_id=None, precision=a.precision or "baseline", | |
| log_fn=lambda x: None) | |
| apis = [] | |
| for _ in range(a.iters): | |
| t0 = time.perf_counter() | |
| api_model(images, instr, state=state, seed=0) | |
| apis.append((time.perf_counter() - t0) * 1e3) | |
| out["api_call_ms"] = med(apis) | |
| out["api_call_ms_p10_p90"] = [float(np.percentile(apis, 10)), float(np.percentile(apis, 90))] | |
| if not a.no_stages: | |
| ns_full = m.read_action_normalized(L0) | |
| out["stages_by_bucket"] = {} | |
| for L in m.buckets: | |
| h = m.host_inputs(images, instr, state, noise, bucket=L) | |
| tids = m.capture_stages(L) | |
| m.write_inputs(h) | |
| for name, tid in tids.items(): | |
| ttnn.execute_trace(dev, tid, cq_id=0, blocking=True) | |
| st = m.time_stages(a.iters) | |
| if "expert" in st: # megakernel: the whole action head is one stage (prologue matmul + program) | |
| r = dict(stage_ms=st, vision_ms=st["vision"], prefix_ms=st["prefix"], expert_total_ms=st["expert"], | |
| stage_sum_ms=sum(st.values())) | |
| else: | |
| steps = [st[f"step{k}"] for k in range(m.num_steps)] | |
| r = dict(stage_ms=st, vision_ms=st["vision"], prefix_ms=st["prefix"], | |
| expert_prologue_ms=st["prologue"], expert_per_step_ms=float(np.mean(steps)), | |
| expert_total_ms=st["prologue"] + sum(steps), stage_sum_ms=sum(st.values())) | |
| if L == L0: | |
| r["stage_chain_bitexact_vs_full"] = bool(torch.equal(m.read_stage_action(), ns_full)) | |
| out.update(r) | |
| out["stages_by_bucket"][str(L)] = r | |
| m.release_stage_traces() | |
| print(json.dumps(out, indent=1), flush=True) | |
| finally: | |
| if a.json: | |
| with open(a.json, "w") as f: | |
| json.dump(out, f, indent=1) | |
| from tt_rdt2.ttaw.device import close_device | |
| close_device(dev) | |
| if __name__ == "__main__": | |
| main() | |