File size: 3,972 Bytes
c869f1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
#!/usr/bin/env python
# SPDX-License-Identifier: Apache-2.0
"""Latency of the RDT2-VQ profile on one p150 (all prompt buckets resident, traced): per bucket the prefill trace and
the decode-step trace (blocking replays), the RVQ decoder, and whole calls through the Python API (host pre-processing,
the 28-token generation loop, the RVQ decode, the unnormalization).

    $ROOT/bin/devrun -t 1500 -- python -u code/scripts/vq_bench.py --iters 100 --json out.json [--dispatch worker]

Environment: RDT2_VQ_RVQ (device | host), RDT2_VQ_DECODE_W (bf16 | bfp8), RDT2_VQ_LOGITS (fp32 | bf16).
"""
from __future__ import annotations

import argparse
import json
import os
import statistics
import sys
import time

sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))


def med(xs):
    return statistics.median(xs)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--iters", type=int, default=100)
    ap.add_argument("--calls", type=int, default=30)
    ap.add_argument("--dispatch", default=os.environ.get("RDT2_DISPATCH", "eth"))
    ap.add_argument("--json", default=None)
    args = ap.parse_args()
    import ttnn

    from tt_rdt2.api import load_sample
    from tt_rdt2.reference.config import DEFAULT_DIRS
    from tt_rdt2.vq.api import RDT2VQ

    wd = {}
    rvq = os.environ.get("RDT2_RVQ_DIR") or os.path.join(os.path.dirname(DEFAULT_DIRS["normalizer"]),
                                                         "hf_RVQActionTokenizer_a6a614ae")
    for role, d in (("vlm", DEFAULT_DIRS["vlm"]), ("normalizer", DEFAULT_DIRS["normalizer"]),
                    ("processor", DEFAULT_DIRS["processor"]), ("rvq", rvq)):
        if os.path.isdir(d):
            wd[role] = d
    t0 = time.time()
    model = RDT2VQ.from_pretrained(dispatch=args.dispatch, weights_dirs=wd, log_fn=print)
    boot = time.time() - t0
    tt, dev = model._m, model.device
    res = {"dispatch": args.dispatch, "boot_s": boot, "grid": model.device_info.get("grid"),
           "decode_w": tt.decode_w, "rvq": tt.rvq_engine, "logits": str(tt.logits_dtype), "buckets": {}}
    s = load_sample()
    with model._lock:
        for L in tt.buckets:
            tt.check_allocator()
            pre, dec = [], []
            for _ in range(args.iters):
                a = time.perf_counter()
                ttnn.execute_trace(dev, tt.traces[("prefill", L)], cq_id=0, blocking=True)
                pre.append((time.perf_counter() - a) * 1e3)
            for _ in range(args.iters):
                a = time.perf_counter()
                ttnn.execute_trace(dev, tt.traces["decode"], cq_id=0, blocking=True)
                dec.append((time.perf_counter() - a) * 1e3)
            res["buckets"][L] = {"prefill_ms": med(pre), "decode_step_ms": med(dec)}
            print(f"bucket {L}: prefill {med(pre):.2f} ms, decode step {med(dec):.2f} ms", flush=True)
        if tt.rvq_engine == "device":
            import torch

            rv = []
            tok = torch.zeros(1, 27, dtype=torch.long)
            for _ in range(args.iters):
                a = time.perf_counter()
                tt.rvq.decode(tok)
                rv.append((time.perf_counter() - a) * 1e3)
            res["rvq_decode_ms"] = med(rv)
    calls, parts = [], []
    for _ in range(args.calls):
        a = time.perf_counter()
        out = model(s["images"], s["instruction"])
        calls.append((time.perf_counter() - a) * 1e3)
        parts.append(out.timing_ms)
    res["call_ms"] = {"median": med(calls), "min": min(calls), "max": max(calls), "n": len(calls)}
    res["call_parts_median_ms"] = {k: med([p[k] for p in parts]) for k in parts[0]}
    res["tokens"] = len(out.generated)
    res["slow_path"] = dict(tt.stats)
    res["host_load"] = os.getloadavg()
    print(json.dumps(res, indent=1), flush=True)
    if args.json:
        with open(args.json, "w") as f:
            json.dump(res, f, indent=1)
    model.close()


if __name__ == "__main__":
    main()