hamer-p150 / code /scripts /bench_p150.py
changh95's picture
tt-model push hamer-p150 (container)
7e9b580 verified
Raw History Blame Contribute Delete
7.09 kB
# SPDX-License-Identifier: Apache-2.0
"""In-process speed benchmark of the published configuration on one p150 (run it in the package image).
python -u /opt/tt-metal/scripts/bench_p150.py --media /media --iters 500 --rounds 20 --json /out/bench_p150.json
It builds the model exactly like the Python API (``HamerModel.from_pretrained``: published serve configuration, ETH
dispatch, 1 command queue, 12x10 grid, warm-up accuracy gate) and measures, batch 1, one hand per call:
device_b2b back-to-back replays of the captured trace (non-blocking, one synchronize per round) / replays
per round: the device time of one forward (ViT-H/16 backbone + MANO head); p50 / p99 over rounds
replay_blocking one trace replay + synchronize, n samples
model_call ``model.run_crop(uint8 crop)``: host upload + trace replay + readback + host finalize, a different
crop every call (the 8 hands of ``media/bboxes.json``, cropped like the server does)
api_array ``model(uint8 RGB array, bbox=..., is_right=...)``: crop + resize + model call + camera; no MANO
api_path ``model("sample1.jpg", bbox=...)``: the same plus the JPEG decode
The weights come from the Hugging Face cache (``changh95/hamer-weights`` at the pinned revision).
"""
from __future__ import annotations
import argparse
import json
import os
import platform
import statistics
import sys
import time
from pathlib import Path
import numpy as np
def pct(v, q):
s = sorted(v)
return s[min(len(s) - 1, max(0, int(round(q / 100.0 * (len(s) - 1)))))]
def summ(v):
return {"n": len(v), "p50": statistics.median(v), "p99": pct(v, 99), "mean": statistics.fmean(v),
"min": min(v), "max": max(v)}
def load_hands(media: Path):
boxes = json.loads((media / "bboxes.json").read_text())
hands = []
for name in sorted(boxes):
ents = boxes[name]
ents = ents if isinstance(ents, list) else [ents]
for e in ents:
if isinstance(e, dict):
hands.append((media / name, [float(v) for v in e["bbox"]], bool(e.get("is_right", 1))))
else:
hands.append((media / name, [float(v) for v in e], True))
return hands
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--media", required=True, help="directory with bboxes.json and the sample JPEGs (the repo's media/)")
ap.add_argument("--sample", default="sample1.jpg")
ap.add_argument("--iters", type=int, default=500, help="samples per host-timed measurement")
ap.add_argument("--rounds", type=int, default=20, help="rounds of back-to-back replays")
ap.add_argument("--replays", type=int, default=200, help="replays per round")
ap.add_argument("--warmup", type=int, default=20)
ap.add_argument("--json", default=None)
a = ap.parse_args()
import torch
import ttnn
from PIL import Image
from hamer import HamerModel
from hamer.preprocess import preprocess_pil
media = Path(a.media)
hands = load_hands(media)
t0 = time.perf_counter()
model = HamerModel.from_pretrained(device_id=0)
boot_s = time.perf_counter() - t0
info = {k: v for k, v in model.info.items() if k in ("dispatch", "num_command_queues", "grid", "warmup",
"native_preprocess", "boot_seconds", "weights")}
print(f"[bench] boot {boot_s:.1f}s info={json.dumps(info)}", flush=True)
res = {"info": info, "boot_s": boot_s, "host": platform.node(), "loadavg_start": os.getloadavg(),
"env": {k: os.environ.get(k) for k in ("HAMER_DISPATCH", "TT_FUSED", "TT_FUSED_INPUT", "TT_FUSED_HASH",
"HAMER_TRACE", "TT_WEIGHTS_REVISION")},
"args": vars(a)}
tt = model.tt
dev = tt.device
trace_id, _ = tt._trace # noqa: SLF001 - the captured trace of the published graph
# 1. device time: back-to-back replays
for _ in range(a.warmup):
ttnn.execute_trace(dev, trace_id, cq_id=0, blocking=False)
ttnn.synchronize_device(dev)
rounds = []
for _ in range(a.rounds):
c0 = time.perf_counter()
for _ in range(a.replays):
ttnn.execute_trace(dev, trace_id, cq_id=0, blocking=False)
ttnn.synchronize_device(dev)
rounds.append((time.perf_counter() - c0) / a.replays * 1e3)
res["device_b2b_ms"] = summ(rounds)
print(f"[bench] device b2b per forward: p50 {res['device_b2b_ms']['p50']:.4f} ms p99 {res['device_b2b_ms']['p99']:.4f} "
f"({a.rounds} rounds x {a.replays})", flush=True)
# 2. one replay + synchronize
blk = []
for _ in range(a.iters):
c0 = time.perf_counter()
ttnn.execute_trace(dev, trace_id, cq_id=0, blocking=False)
ttnn.synchronize_device(dev)
blk.append((time.perf_counter() - c0) * 1e3)
res["replay_blocking_ms"] = summ(blk)
# 3. model call on uint8 crops (what the server hands the model), a different crop every call
crops = []
for path, bbox, right in hands:
img = Image.open(path).convert("RGB")
x, _ = preprocess_pil(img, tuple(bbox), rescale_factor=2.0, is_right=right, raw=model.raw_input,
contiguous=not model.raw_input)
crops.append(x)
for i in range(a.warmup):
model.run_crop(crops[i % len(crops)])
mc = []
for i in range(a.iters):
x = crops[i % len(crops)]
c0 = time.perf_counter()
model.run_crop(x)
mc.append((time.perf_counter() - c0) * 1e3)
res["model_call_ms"] = summ(mc)
# 4. Python API on the sample frame
sp = media / a.sample
boxes = json.loads((media / "bboxes.json").read_text())
e = boxes[a.sample][0]
bbox, right = [float(v) for v in e["bbox"]], bool(e.get("is_right", 1))
arr = np.asarray(Image.open(sp).convert("RGB"))
res["sample"] = {"file": a.sample, "size": list(Image.open(sp).size), "bbox": bbox, "is_right": right,
"bytes": sp.stat().st_size}
for name, src in (("api_array_ms", arr), ("api_path_ms", str(sp))):
for _ in range(a.warmup):
model(src, bbox=bbox, is_right=right)
v = []
for _ in range(a.iters):
c0 = time.perf_counter()
hand = model(src, bbox=bbox, is_right=right)
v.append((time.perf_counter() - c0) * 1e3)
res[name] = summ(v)
res["sample_output"] = hand.to_dict()
res["loadavg_end"] = os.getloadavg()
for k in ("replay_blocking_ms", "model_call_ms", "api_array_ms", "api_path_ms"):
s = res[k]
print(f"[bench] {k}: p50 {s['p50']:.3f} p99 {s['p99']:.3f} min {s['min']:.3f} (n={s['n']})", flush=True)
print("RESULT " + json.dumps({k: res[k] for k in ("device_b2b_ms", "replay_blocking_ms", "model_call_ms",
"api_array_ms", "api_path_ms")}), flush=True)
if a.json:
Path(a.json).write_text(json.dumps(res, indent=1))
model.close()
if __name__ == "__main__":
sys.exit(main())