File size: 8,489 Bytes
9d8d25c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 | # SPDX-License-Identifier: Apache-2.0
"""Latency of RDT2-FM (engine per program: RDT2_ENGINE, e.g. expert=mk): trace replay per call, per stage, per expert step, host pre / post, H2D / D2H.
python -u code/scripts/bench.py --iters 40 --json bench.json # from the repo root (one chip; ETH)
Weights: the ``RDT2_*_DIR`` directories, else the pinned snapshots in the Hugging Face cache (``tt_rdt2.api
.resolve_weights``: the same resolution as the server). Also measures the Python API call (``RDT2FM.__call__``: request
validation + the call; ``api_call_ms``) on the same resident traces.
Input: the shipped Apache-2.0 sample (code/tt_rdt2/samples/ur5e_bimanual_example_0.npz) = golden real_ur5e__s0.
All ``--buckets`` are resident (one trace each); the sample's own bucket (416) gives the headline numbers and the same
request is also forced into every resident bucket (``trace_replay_ms_by_bucket``, ``stages_by_bucket``).
Each number is the median of ``--iters`` repetitions after 5 warm-up replays; device times are host wall-clock around
``execute_trace(blocking=False)`` + ``synchronize_device`` (one call at a time, batch 1). Every replay first runs the
memory-contract check (``check_persistent``: persistent addresses + allocator totals, about 0.6 ms of host time), so it
is inside the trace-replay, stage and e2e numbers.
"""
from __future__ import annotations
import argparse
import json
import os
import statistics
import sys
import time
import numpy as np
import torch
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
from tt_rdt2.tt.model import RDT2FMTT, open_rdt2_device # noqa: E402
SAMPLE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "tt_rdt2", "samples",
"ur5e_bimanual_example_0.npz")
def med(xs):
return float(statistics.median(xs))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--precision", default=None)
ap.add_argument("--buckets", default="416,448,512", help="resident prompt buckets (one trace each)")
ap.add_argument("--dispatch", default="eth")
ap.add_argument("--iters", type=int, default=60)
ap.add_argument("--json", default=None)
ap.add_argument("--no-stages", action="store_true")
a = ap.parse_args()
import ttnn
s = np.load(SAMPLE, allow_pickle=False)
images = [s["left_stereo"], s["right_stereo"]] if "left_stereo" in s.files else [s["left"], s["right"]]
instr = str(s["instruction"]) if "instruction" in s.files else "Pour water from the plastic bottle into the cup"
state = s["state"] if "state" in s.files else np.zeros(20, np.float32)
noise = torch.randn((1, 24, 20), generator=torch.Generator().manual_seed(0))
dev = open_rdt2_device(a.dispatch)
g = dev.compute_with_storage_grid_size()
buckets = tuple(int(b) for b in a.buckets.split(","))
out = {"grid": f"{g.x}x{g.y}", "dispatch": a.dispatch, "buckets": list(buckets), "iters": a.iters,
"sample": os.path.basename(SAMPLE), "torch_threads": torch.get_num_threads()}
try:
from tt_rdt2.api import RDT2FM, resolve_weights
from tt_rdt2.reference.config import DEFAULT_DIRS
w = resolve_weights(weights_dirs={r: d for r, d in DEFAULT_DIRS.items() if r != "golden" and os.path.isdir(d)},
log_fn=lambda x: None)
m = RDT2FMTT(dev, buckets=buckets, precision=a.precision, vlm_dir=w["vlm"]["path"],
expert_dir=w["expert"]["path"], normalizer_dir=w["normalizer"]["path"],
processor_dir=w["processor"]["path"])
out["precision"] = m.prec.as_dict()
out["load_s"] = m.timings["load_s"]
# host pre
hp = []
for _ in range(a.iters):
t0 = time.perf_counter()
host = m.host_inputs(images, instr, state, noise)
hp.append((time.perf_counter() - t0) * 1e3)
out["host_pre_ms"] = med(hp)
L0 = host["bucket"]
out["bucket"] = L0
out["n_tokens"] = int(host["n_tokens"])
t0 = time.perf_counter()
m.capture()
out["capture_s_all_buckets"] = time.perf_counter() - t0
# H2D (5 input buffers)
h2d = []
for _ in range(a.iters):
ttnn.synchronize_device(dev)
t0 = time.perf_counter()
m.write_inputs(host)
ttnn.synchronize_device(dev)
h2d.append((time.perf_counter() - t0) * 1e3)
out["h2d_ms"] = med(h2d)
def time_replay(L):
for _ in range(5):
m.replay(L, blocking=True)
rep = []
for _ in range(a.iters):
t0 = time.perf_counter()
m.replay(L, blocking=False)
ttnn.synchronize_device(dev)
rep.append((time.perf_counter() - t0) * 1e3)
return rep
by_bucket = {}
for L in m.buckets: # the same request forced into every resident bucket
m.write_inputs(m.host_inputs(images, instr, state, noise, bucket=L))
by_bucket[str(L)] = med(time_replay(L))
out["trace_replay_ms_by_bucket"] = by_bucket
m.write_inputs(host)
rep = time_replay(L0)
out["trace_replay_ms"] = med(rep)
out["trace_replay_ms_p10_p90"] = [float(np.percentile(rep, 10)), float(np.percentile(rep, 90))]
d2h = []
for _ in range(a.iters):
t0 = time.perf_counter()
ns = m.read_action_normalized(L0)
d2h.append((time.perf_counter() - t0) * 1e3)
out["d2h_ms"] = med(d2h)
from tt_rdt2.reference.expert import unnormalize_action
hpost = []
for _ in range(a.iters):
t0 = time.perf_counter()
unnormalize_action(ns, m.norm)
hpost.append((time.perf_counter() - t0) * 1e3)
out["host_post_ms"] = med(hpost)
e2e = []
for _ in range(a.iters):
t0 = time.perf_counter()
r = m(images, instr, state, noise)
e2e.append((time.perf_counter() - t0) * 1e3)
out["e2e_call_ms"] = med(e2e)
out["e2e_call_ms_p10_p90"] = [float(np.percentile(e2e, 10)), float(np.percentile(e2e, 90))]
out["action_first_row"] = r["action"][0].tolist()
api_model = RDT2FM(m, dev, weights=w, owns_device=False, device_id=None, precision=a.precision or "baseline",
log_fn=lambda x: None)
apis = []
for _ in range(a.iters):
t0 = time.perf_counter()
api_model(images, instr, state=state, seed=0)
apis.append((time.perf_counter() - t0) * 1e3)
out["api_call_ms"] = med(apis)
out["api_call_ms_p10_p90"] = [float(np.percentile(apis, 10)), float(np.percentile(apis, 90))]
if not a.no_stages:
ns_full = m.read_action_normalized(L0)
out["stages_by_bucket"] = {}
for L in m.buckets:
h = m.host_inputs(images, instr, state, noise, bucket=L)
tids = m.capture_stages(L)
m.write_inputs(h)
for name, tid in tids.items():
ttnn.execute_trace(dev, tid, cq_id=0, blocking=True)
st = m.time_stages(a.iters)
if "expert" in st: # megakernel: the whole action head is one stage (prologue matmul + program)
r = dict(stage_ms=st, vision_ms=st["vision"], prefix_ms=st["prefix"], expert_total_ms=st["expert"],
stage_sum_ms=sum(st.values()))
else:
steps = [st[f"step{k}"] for k in range(m.num_steps)]
r = dict(stage_ms=st, vision_ms=st["vision"], prefix_ms=st["prefix"],
expert_prologue_ms=st["prologue"], expert_per_step_ms=float(np.mean(steps)),
expert_total_ms=st["prologue"] + sum(steps), stage_sum_ms=sum(st.values()))
if L == L0:
r["stage_chain_bitexact_vs_full"] = bool(torch.equal(m.read_stage_action(), ns_full))
out.update(r)
out["stages_by_bucket"][str(L)] = r
m.release_stage_traces()
print(json.dumps(out, indent=1), flush=True)
finally:
if a.json:
with open(a.json, "w") as f:
json.dump(out, f, indent=1)
from tt_rdt2.ttaw.device import close_device
close_device(dev)
if __name__ == "__main__":
main()
|