File size: 8,489 Bytes
9d8d25c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
# SPDX-License-Identifier: Apache-2.0
"""Latency of RDT2-FM (engine per program: RDT2_ENGINE, e.g. expert=mk): trace replay per call, per stage, per expert step, host pre / post, H2D / D2H.

    python -u code/scripts/bench.py --iters 40 --json bench.json            # from the repo root (one chip; ETH)

Weights: the ``RDT2_*_DIR`` directories, else the pinned snapshots in the Hugging Face cache (``tt_rdt2.api
.resolve_weights``: the same resolution as the server). Also measures the Python API call (``RDT2FM.__call__``: request
validation + the call; ``api_call_ms``) on the same resident traces.

Input: the shipped Apache-2.0 sample (code/tt_rdt2/samples/ur5e_bimanual_example_0.npz) = golden real_ur5e__s0.
All ``--buckets`` are resident (one trace each); the sample's own bucket (416) gives the headline numbers and the same
request is also forced into every resident bucket (``trace_replay_ms_by_bucket``, ``stages_by_bucket``).
Each number is the median of ``--iters`` repetitions after 5 warm-up replays; device times are host wall-clock around
``execute_trace(blocking=False)`` + ``synchronize_device`` (one call at a time, batch 1). Every replay first runs the
memory-contract check (``check_persistent``: persistent addresses + allocator totals, about 0.6 ms of host time), so it
is inside the trace-replay, stage and e2e numbers.
"""
from __future__ import annotations

import argparse
import json
import os
import statistics
import sys
import time

import numpy as np
import torch

sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))

from tt_rdt2.tt.model import RDT2FMTT, open_rdt2_device  # noqa: E402

SAMPLE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "tt_rdt2", "samples",
                      "ur5e_bimanual_example_0.npz")


def med(xs):
    return float(statistics.median(xs))


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--precision", default=None)
    ap.add_argument("--buckets", default="416,448,512", help="resident prompt buckets (one trace each)")
    ap.add_argument("--dispatch", default="eth")
    ap.add_argument("--iters", type=int, default=60)
    ap.add_argument("--json", default=None)
    ap.add_argument("--no-stages", action="store_true")
    a = ap.parse_args()
    import ttnn

    s = np.load(SAMPLE, allow_pickle=False)
    images = [s["left_stereo"], s["right_stereo"]] if "left_stereo" in s.files else [s["left"], s["right"]]
    instr = str(s["instruction"]) if "instruction" in s.files else "Pour water from the plastic bottle into the cup"
    state = s["state"] if "state" in s.files else np.zeros(20, np.float32)
    noise = torch.randn((1, 24, 20), generator=torch.Generator().manual_seed(0))
    dev = open_rdt2_device(a.dispatch)
    g = dev.compute_with_storage_grid_size()
    buckets = tuple(int(b) for b in a.buckets.split(","))
    out = {"grid": f"{g.x}x{g.y}", "dispatch": a.dispatch, "buckets": list(buckets), "iters": a.iters,
           "sample": os.path.basename(SAMPLE), "torch_threads": torch.get_num_threads()}
    try:
        from tt_rdt2.api import RDT2FM, resolve_weights
        from tt_rdt2.reference.config import DEFAULT_DIRS

        w = resolve_weights(weights_dirs={r: d for r, d in DEFAULT_DIRS.items() if r != "golden" and os.path.isdir(d)},
                            log_fn=lambda x: None)
        m = RDT2FMTT(dev, buckets=buckets, precision=a.precision, vlm_dir=w["vlm"]["path"],
                     expert_dir=w["expert"]["path"], normalizer_dir=w["normalizer"]["path"],
                     processor_dir=w["processor"]["path"])
        out["precision"] = m.prec.as_dict()
        out["load_s"] = m.timings["load_s"]
        # host pre
        hp = []
        for _ in range(a.iters):
            t0 = time.perf_counter()
            host = m.host_inputs(images, instr, state, noise)
            hp.append((time.perf_counter() - t0) * 1e3)
        out["host_pre_ms"] = med(hp)
        L0 = host["bucket"]
        out["bucket"] = L0
        out["n_tokens"] = int(host["n_tokens"])
        t0 = time.perf_counter()
        m.capture()
        out["capture_s_all_buckets"] = time.perf_counter() - t0
        # H2D (5 input buffers)
        h2d = []
        for _ in range(a.iters):
            ttnn.synchronize_device(dev)
            t0 = time.perf_counter()
            m.write_inputs(host)
            ttnn.synchronize_device(dev)
            h2d.append((time.perf_counter() - t0) * 1e3)
        out["h2d_ms"] = med(h2d)
        def time_replay(L):
            for _ in range(5):
                m.replay(L, blocking=True)
            rep = []
            for _ in range(a.iters):
                t0 = time.perf_counter()
                m.replay(L, blocking=False)
                ttnn.synchronize_device(dev)
                rep.append((time.perf_counter() - t0) * 1e3)
            return rep

        by_bucket = {}
        for L in m.buckets:  # the same request forced into every resident bucket
            m.write_inputs(m.host_inputs(images, instr, state, noise, bucket=L))
            by_bucket[str(L)] = med(time_replay(L))
        out["trace_replay_ms_by_bucket"] = by_bucket
        m.write_inputs(host)
        rep = time_replay(L0)
        out["trace_replay_ms"] = med(rep)
        out["trace_replay_ms_p10_p90"] = [float(np.percentile(rep, 10)), float(np.percentile(rep, 90))]
        d2h = []
        for _ in range(a.iters):
            t0 = time.perf_counter()
            ns = m.read_action_normalized(L0)
            d2h.append((time.perf_counter() - t0) * 1e3)
        out["d2h_ms"] = med(d2h)
        from tt_rdt2.reference.expert import unnormalize_action

        hpost = []
        for _ in range(a.iters):
            t0 = time.perf_counter()
            unnormalize_action(ns, m.norm)
            hpost.append((time.perf_counter() - t0) * 1e3)
        out["host_post_ms"] = med(hpost)
        e2e = []
        for _ in range(a.iters):
            t0 = time.perf_counter()
            r = m(images, instr, state, noise)
            e2e.append((time.perf_counter() - t0) * 1e3)
        out["e2e_call_ms"] = med(e2e)
        out["e2e_call_ms_p10_p90"] = [float(np.percentile(e2e, 10)), float(np.percentile(e2e, 90))]
        out["action_first_row"] = r["action"][0].tolist()
        api_model = RDT2FM(m, dev, weights=w, owns_device=False, device_id=None, precision=a.precision or "baseline",
                           log_fn=lambda x: None)
        apis = []
        for _ in range(a.iters):
            t0 = time.perf_counter()
            api_model(images, instr, state=state, seed=0)
            apis.append((time.perf_counter() - t0) * 1e3)
        out["api_call_ms"] = med(apis)
        out["api_call_ms_p10_p90"] = [float(np.percentile(apis, 10)), float(np.percentile(apis, 90))]
        if not a.no_stages:
            ns_full = m.read_action_normalized(L0)
            out["stages_by_bucket"] = {}
            for L in m.buckets:
                h = m.host_inputs(images, instr, state, noise, bucket=L)
                tids = m.capture_stages(L)
                m.write_inputs(h)
                for name, tid in tids.items():
                    ttnn.execute_trace(dev, tid, cq_id=0, blocking=True)
                st = m.time_stages(a.iters)
                if "expert" in st:  # megakernel: the whole action head is one stage (prologue matmul + program)
                    r = dict(stage_ms=st, vision_ms=st["vision"], prefix_ms=st["prefix"], expert_total_ms=st["expert"],
                             stage_sum_ms=sum(st.values()))
                else:
                    steps = [st[f"step{k}"] for k in range(m.num_steps)]
                    r = dict(stage_ms=st, vision_ms=st["vision"], prefix_ms=st["prefix"],
                             expert_prologue_ms=st["prologue"], expert_per_step_ms=float(np.mean(steps)),
                             expert_total_ms=st["prologue"] + sum(steps), stage_sum_ms=sum(st.values()))
                if L == L0:
                    r["stage_chain_bitexact_vs_full"] = bool(torch.equal(m.read_stage_action(), ns_full))
                    out.update(r)
                out["stages_by_bucket"][str(L)] = r
                m.release_stage_traces()
        print(json.dumps(out, indent=1), flush=True)
    finally:
        if a.json:
            with open(a.json, "w") as f:
                json.dump(out, f, indent=1)
        from tt_rdt2.ttaw.device import close_device

        close_device(dev)


if __name__ == "__main__":
    main()