changh95's picture
tt-model push rdt2-base-p150 (container)
9d8d25c verified
Raw History Blame Contribute Delete
8.49 kB
# SPDX-License-Identifier: Apache-2.0
"""Latency of RDT2-FM (engine per program: RDT2_ENGINE, e.g. expert=mk): trace replay per call, per stage, per expert step, host pre / post, H2D / D2H.
python -u code/scripts/bench.py --iters 40 --json bench.json # from the repo root (one chip; ETH)
Weights: the ``RDT2_*_DIR`` directories, else the pinned snapshots in the Hugging Face cache (``tt_rdt2.api
.resolve_weights``: the same resolution as the server). Also measures the Python API call (``RDT2FM.__call__``: request
validation + the call; ``api_call_ms``) on the same resident traces.
Input: the shipped Apache-2.0 sample (code/tt_rdt2/samples/ur5e_bimanual_example_0.npz) = golden real_ur5e__s0.
All ``--buckets`` are resident (one trace each); the sample's own bucket (416) gives the headline numbers and the same
request is also forced into every resident bucket (``trace_replay_ms_by_bucket``, ``stages_by_bucket``).
Each number is the median of ``--iters`` repetitions after 5 warm-up replays; device times are host wall-clock around
``execute_trace(blocking=False)`` + ``synchronize_device`` (one call at a time, batch 1). Every replay first runs the
memory-contract check (``check_persistent``: persistent addresses + allocator totals, about 0.6 ms of host time), so it
is inside the trace-replay, stage and e2e numbers.
"""
from __future__ import annotations
import argparse
import json
import os
import statistics
import sys
import time
import numpy as np
import torch
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
from tt_rdt2.tt.model import RDT2FMTT, open_rdt2_device # noqa: E402
SAMPLE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "tt_rdt2", "samples",
"ur5e_bimanual_example_0.npz")
def med(xs):
return float(statistics.median(xs))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--precision", default=None)
ap.add_argument("--buckets", default="416,448,512", help="resident prompt buckets (one trace each)")
ap.add_argument("--dispatch", default="eth")
ap.add_argument("--iters", type=int, default=60)
ap.add_argument("--json", default=None)
ap.add_argument("--no-stages", action="store_true")
a = ap.parse_args()
import ttnn
s = np.load(SAMPLE, allow_pickle=False)
images = [s["left_stereo"], s["right_stereo"]] if "left_stereo" in s.files else [s["left"], s["right"]]
instr = str(s["instruction"]) if "instruction" in s.files else "Pour water from the plastic bottle into the cup"
state = s["state"] if "state" in s.files else np.zeros(20, np.float32)
noise = torch.randn((1, 24, 20), generator=torch.Generator().manual_seed(0))
dev = open_rdt2_device(a.dispatch)
g = dev.compute_with_storage_grid_size()
buckets = tuple(int(b) for b in a.buckets.split(","))
out = {"grid": f"{g.x}x{g.y}", "dispatch": a.dispatch, "buckets": list(buckets), "iters": a.iters,
"sample": os.path.basename(SAMPLE), "torch_threads": torch.get_num_threads()}
try:
from tt_rdt2.api import RDT2FM, resolve_weights
from tt_rdt2.reference.config import DEFAULT_DIRS
w = resolve_weights(weights_dirs={r: d for r, d in DEFAULT_DIRS.items() if r != "golden" and os.path.isdir(d)},
log_fn=lambda x: None)
m = RDT2FMTT(dev, buckets=buckets, precision=a.precision, vlm_dir=w["vlm"]["path"],
expert_dir=w["expert"]["path"], normalizer_dir=w["normalizer"]["path"],
processor_dir=w["processor"]["path"])
out["precision"] = m.prec.as_dict()
out["load_s"] = m.timings["load_s"]
# host pre
hp = []
for _ in range(a.iters):
t0 = time.perf_counter()
host = m.host_inputs(images, instr, state, noise)
hp.append((time.perf_counter() - t0) * 1e3)
out["host_pre_ms"] = med(hp)
L0 = host["bucket"]
out["bucket"] = L0
out["n_tokens"] = int(host["n_tokens"])
t0 = time.perf_counter()
m.capture()
out["capture_s_all_buckets"] = time.perf_counter() - t0
# H2D (5 input buffers)
h2d = []
for _ in range(a.iters):
ttnn.synchronize_device(dev)
t0 = time.perf_counter()
m.write_inputs(host)
ttnn.synchronize_device(dev)
h2d.append((time.perf_counter() - t0) * 1e3)
out["h2d_ms"] = med(h2d)
def time_replay(L):
for _ in range(5):
m.replay(L, blocking=True)
rep = []
for _ in range(a.iters):
t0 = time.perf_counter()
m.replay(L, blocking=False)
ttnn.synchronize_device(dev)
rep.append((time.perf_counter() - t0) * 1e3)
return rep
by_bucket = {}
for L in m.buckets: # the same request forced into every resident bucket
m.write_inputs(m.host_inputs(images, instr, state, noise, bucket=L))
by_bucket[str(L)] = med(time_replay(L))
out["trace_replay_ms_by_bucket"] = by_bucket
m.write_inputs(host)
rep = time_replay(L0)
out["trace_replay_ms"] = med(rep)
out["trace_replay_ms_p10_p90"] = [float(np.percentile(rep, 10)), float(np.percentile(rep, 90))]
d2h = []
for _ in range(a.iters):
t0 = time.perf_counter()
ns = m.read_action_normalized(L0)
d2h.append((time.perf_counter() - t0) * 1e3)
out["d2h_ms"] = med(d2h)
from tt_rdt2.reference.expert import unnormalize_action
hpost = []
for _ in range(a.iters):
t0 = time.perf_counter()
unnormalize_action(ns, m.norm)
hpost.append((time.perf_counter() - t0) * 1e3)
out["host_post_ms"] = med(hpost)
e2e = []
for _ in range(a.iters):
t0 = time.perf_counter()
r = m(images, instr, state, noise)
e2e.append((time.perf_counter() - t0) * 1e3)
out["e2e_call_ms"] = med(e2e)
out["e2e_call_ms_p10_p90"] = [float(np.percentile(e2e, 10)), float(np.percentile(e2e, 90))]
out["action_first_row"] = r["action"][0].tolist()
api_model = RDT2FM(m, dev, weights=w, owns_device=False, device_id=None, precision=a.precision or "baseline",
log_fn=lambda x: None)
apis = []
for _ in range(a.iters):
t0 = time.perf_counter()
api_model(images, instr, state=state, seed=0)
apis.append((time.perf_counter() - t0) * 1e3)
out["api_call_ms"] = med(apis)
out["api_call_ms_p10_p90"] = [float(np.percentile(apis, 10)), float(np.percentile(apis, 90))]
if not a.no_stages:
ns_full = m.read_action_normalized(L0)
out["stages_by_bucket"] = {}
for L in m.buckets:
h = m.host_inputs(images, instr, state, noise, bucket=L)
tids = m.capture_stages(L)
m.write_inputs(h)
for name, tid in tids.items():
ttnn.execute_trace(dev, tid, cq_id=0, blocking=True)
st = m.time_stages(a.iters)
if "expert" in st: # megakernel: the whole action head is one stage (prologue matmul + program)
r = dict(stage_ms=st, vision_ms=st["vision"], prefix_ms=st["prefix"], expert_total_ms=st["expert"],
stage_sum_ms=sum(st.values()))
else:
steps = [st[f"step{k}"] for k in range(m.num_steps)]
r = dict(stage_ms=st, vision_ms=st["vision"], prefix_ms=st["prefix"],
expert_prologue_ms=st["prologue"], expert_per_step_ms=float(np.mean(steps)),
expert_total_ms=st["prologue"] + sum(steps), stage_sum_ms=sum(st.values()))
if L == L0:
r["stage_chain_bitexact_vs_full"] = bool(torch.equal(m.read_stage_action(), ns_full))
out.update(r)
out["stages_by_bucket"][str(L)] = r
m.release_stage_traces()
print(json.dumps(out, indent=1), flush=True)
finally:
if a.json:
with open(a.json, "w") as f:
json.dump(out, f, indent=1)
from tt_rdt2.ttaw.device import close_device
close_device(dev)
if __name__ == "__main__":
main()