# SPDX-License-Identifier: Apache-2.0 """In-process speed benchmark of the published configuration on one p150 (run it in the package image). python -u /opt/tt-metal/scripts/bench_p150.py --media /media --iters 500 --rounds 20 --json /out/bench_p150.json It builds the model exactly like the Python API (``HamerModel.from_pretrained``: published serve configuration, ETH dispatch, 1 command queue, 12x10 grid, warm-up accuracy gate) and measures, batch 1, one hand per call: device_b2b back-to-back replays of the captured trace (non-blocking, one synchronize per round) / replays per round: the device time of one forward (ViT-H/16 backbone + MANO head); p50 / p99 over rounds replay_blocking one trace replay + synchronize, n samples model_call ``model.run_crop(uint8 crop)``: host upload + trace replay + readback + host finalize, a different crop every call (the 8 hands of ``media/bboxes.json``, cropped like the server does) api_array ``model(uint8 RGB array, bbox=..., is_right=...)``: crop + resize + model call + camera; no MANO api_path ``model("sample1.jpg", bbox=...)``: the same plus the JPEG decode The weights come from the Hugging Face cache (``changh95/hamer-weights`` at the pinned revision). """ from __future__ import annotations import argparse import json import os import platform import statistics import sys import time from pathlib import Path import numpy as np def pct(v, q): s = sorted(v) return s[min(len(s) - 1, max(0, int(round(q / 100.0 * (len(s) - 1)))))] def summ(v): return {"n": len(v), "p50": statistics.median(v), "p99": pct(v, 99), "mean": statistics.fmean(v), "min": min(v), "max": max(v)} def load_hands(media: Path): boxes = json.loads((media / "bboxes.json").read_text()) hands = [] for name in sorted(boxes): ents = boxes[name] ents = ents if isinstance(ents, list) else [ents] for e in ents: if isinstance(e, dict): hands.append((media / name, [float(v) for v in e["bbox"]], bool(e.get("is_right", 1)))) else: hands.append((media / name, [float(v) for v in e], True)) return hands def main(): ap = argparse.ArgumentParser() ap.add_argument("--media", required=True, help="directory with bboxes.json and the sample JPEGs (the repo's media/)") ap.add_argument("--sample", default="sample1.jpg") ap.add_argument("--iters", type=int, default=500, help="samples per host-timed measurement") ap.add_argument("--rounds", type=int, default=20, help="rounds of back-to-back replays") ap.add_argument("--replays", type=int, default=200, help="replays per round") ap.add_argument("--warmup", type=int, default=20) ap.add_argument("--json", default=None) a = ap.parse_args() import torch import ttnn from PIL import Image from hamer import HamerModel from hamer.preprocess import preprocess_pil media = Path(a.media) hands = load_hands(media) t0 = time.perf_counter() model = HamerModel.from_pretrained(device_id=0) boot_s = time.perf_counter() - t0 info = {k: v for k, v in model.info.items() if k in ("dispatch", "num_command_queues", "grid", "warmup", "native_preprocess", "boot_seconds", "weights")} print(f"[bench] boot {boot_s:.1f}s info={json.dumps(info)}", flush=True) res = {"info": info, "boot_s": boot_s, "host": platform.node(), "loadavg_start": os.getloadavg(), "env": {k: os.environ.get(k) for k in ("HAMER_DISPATCH", "TT_FUSED", "TT_FUSED_INPUT", "TT_FUSED_HASH", "HAMER_TRACE", "TT_WEIGHTS_REVISION")}, "args": vars(a)} tt = model.tt dev = tt.device trace_id, _ = tt._trace # noqa: SLF001 - the captured trace of the published graph # 1. device time: back-to-back replays for _ in range(a.warmup): ttnn.execute_trace(dev, trace_id, cq_id=0, blocking=False) ttnn.synchronize_device(dev) rounds = [] for _ in range(a.rounds): c0 = time.perf_counter() for _ in range(a.replays): ttnn.execute_trace(dev, trace_id, cq_id=0, blocking=False) ttnn.synchronize_device(dev) rounds.append((time.perf_counter() - c0) / a.replays * 1e3) res["device_b2b_ms"] = summ(rounds) print(f"[bench] device b2b per forward: p50 {res['device_b2b_ms']['p50']:.4f} ms p99 {res['device_b2b_ms']['p99']:.4f} " f"({a.rounds} rounds x {a.replays})", flush=True) # 2. one replay + synchronize blk = [] for _ in range(a.iters): c0 = time.perf_counter() ttnn.execute_trace(dev, trace_id, cq_id=0, blocking=False) ttnn.synchronize_device(dev) blk.append((time.perf_counter() - c0) * 1e3) res["replay_blocking_ms"] = summ(blk) # 3. model call on uint8 crops (what the server hands the model), a different crop every call crops = [] for path, bbox, right in hands: img = Image.open(path).convert("RGB") x, _ = preprocess_pil(img, tuple(bbox), rescale_factor=2.0, is_right=right, raw=model.raw_input, contiguous=not model.raw_input) crops.append(x) for i in range(a.warmup): model.run_crop(crops[i % len(crops)]) mc = [] for i in range(a.iters): x = crops[i % len(crops)] c0 = time.perf_counter() model.run_crop(x) mc.append((time.perf_counter() - c0) * 1e3) res["model_call_ms"] = summ(mc) # 4. Python API on the sample frame sp = media / a.sample boxes = json.loads((media / "bboxes.json").read_text()) e = boxes[a.sample][0] bbox, right = [float(v) for v in e["bbox"]], bool(e.get("is_right", 1)) arr = np.asarray(Image.open(sp).convert("RGB")) res["sample"] = {"file": a.sample, "size": list(Image.open(sp).size), "bbox": bbox, "is_right": right, "bytes": sp.stat().st_size} for name, src in (("api_array_ms", arr), ("api_path_ms", str(sp))): for _ in range(a.warmup): model(src, bbox=bbox, is_right=right) v = [] for _ in range(a.iters): c0 = time.perf_counter() hand = model(src, bbox=bbox, is_right=right) v.append((time.perf_counter() - c0) * 1e3) res[name] = summ(v) res["sample_output"] = hand.to_dict() res["loadavg_end"] = os.getloadavg() for k in ("replay_blocking_ms", "model_call_ms", "api_array_ms", "api_path_ms"): s = res[k] print(f"[bench] {k}: p50 {s['p50']:.3f} p99 {s['p99']:.3f} min {s['min']:.3f} (n={s['n']})", flush=True) print("RESULT " + json.dumps({k: res[k] for k in ("device_b2b_ms", "replay_blocking_ms", "model_call_ms", "api_array_ms", "api_path_ms")}), flush=True) if a.json: Path(a.json).write_text(json.dumps(res, indent=1)) model.close() if __name__ == "__main__": sys.exit(main())