Download code/scripts/eval_agreement.py from changh95/hamer-p150: direct link, hf CLI and curl.
- Browser
- Download file 9.2 kB
-
https://huggingface.co/changh95/hamer-p150/resolve/main/code/scripts/eval_agreement.py
- Command line
-
hf download hf://changh95/hamer-p150/code/scripts/eval_agreement.py
-
curl -L -o eval_agreement.py https://huggingface.co/changh95/hamer-p150/resolve/main/code/scripts/eval_agreement.py
9.2 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| """Agreement of this package with the fp32 torch CPU reference on real hand images. | |
| Run the same list of (image, hand box, is_right) through one implementation and save the outputs: | |
| # fp32 torch CPU reference (hamer.reference.Hamer with the released checkpoint; no device) | |
| python eval_agreement.py run --impl cpu --samples samples.json --root <image dir> --out ref.npz | |
| # this package through the Python API on the chip (published configuration) | |
| python eval_agreement.py run --impl tt --samples samples.json --root <image dir> --out tt.npz | |
| # this package through a running server (POST /predict, what a client receives) | |
| python eval_agreement.py run --impl http --url http://127.0.0.1:20000 --samples samples.json --root <dir> --out http.npz | |
| and compare two output files: | |
| python eval_agreement.py compare --ref ref.npz --test http.npz --json agreement.json | |
| ``samples.json`` is a list of ``{"id", "set", "image" (path relative to --root), "bbox": [x1, y1, x2, y2], | |
| "is_right"}``. Every implementation does the same host steps (decode, square crop of 2x the box, resize, mirror for a | |
| left hand) with the package's own preprocessing; the CPU reference runs on the normalized float crop and the chip on | |
| the uint8 crop, as served. | |
| Metrics per set (``compare``): PCC of the 157-value regression output (16 rotation matrices, 10 betas, crop camera), | |
| the geodesic angle between corresponding rotation matrices (global orientation and the 15 MANO joint rotations, in | |
| degrees), the absolute betas error, the full-frame camera translation error ``|cam_t_full - ref|`` in millimetres | |
| (MANO units are metres) and its relative depth error. Without a MANO model file the outputs are MANO parameters, so | |
| joint and vertex errors are not computed here. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import base64 | |
| import json | |
| import os | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| def load_samples(path): | |
| return json.loads(Path(path).read_text()) | |
| def run(a): | |
| samples = load_samples(a.samples) | |
| if a.limit: | |
| samples = samples[: a.limit] | |
| root = Path(a.root) | |
| n = len(samples) | |
| out = {"rotmats": np.zeros((n, 16, 3, 3)), "betas": np.zeros((n, 10)), "cam": np.zeros((n, 3)), | |
| "cam_t_full": np.zeros((n, 3)), "time_ms": np.zeros(n)} | |
| t_start = time.time() | |
| if a.impl == "http": | |
| import http.client | |
| import urllib.parse | |
| u = urllib.parse.urlparse(a.url) | |
| conn = http.client.HTTPConnection(u.hostname, u.port or 80, timeout=120) | |
| cache = {} | |
| for i, s in enumerate(samples): | |
| p = root / s["image"] | |
| if p not in cache: | |
| cache = {p: base64.b64encode(p.read_bytes()).decode()} | |
| body = json.dumps({"image": cache[p], "bbox": s["bbox"], "is_right": bool(s["is_right"])}).encode() | |
| t0 = time.perf_counter() | |
| conn.request("POST", "/predict", body=body, headers={"Content-Type": "application/json"}) | |
| r = conn.getresponse() | |
| raw = r.read() | |
| out["time_ms"][i] = (time.perf_counter() - t0) * 1e3 | |
| if r.status != 200: | |
| raise SystemExit(f"{s['id']}: HTTP {r.status} {raw[:300]!r}") | |
| d = json.loads(raw) | |
| for k in ("rotmats", "betas", "cam", "cam_t_full"): | |
| out[k][i] = np.asarray(d[k], dtype=np.float64) | |
| else: | |
| import torch | |
| from PIL import Image | |
| from hamer import HamerModel, load_reference | |
| if a.threads: | |
| torch.set_num_threads(a.threads) | |
| if a.impl == "cpu": | |
| ref = load_reference() | |
| model = HamerModel.from_reference(ref, backend="cpu", warmup_variants=None) | |
| else: | |
| model = HamerModel.from_pretrained(device_id=0, warmup_variants=None) | |
| print(f"[eval] impl={a.impl} info={json.dumps({k: model.info.get(k) for k in ('backend', 'dispatch', 'grid', 'warmup')})}", | |
| flush=True) | |
| img_cache = {} | |
| for i, s in enumerate(samples): | |
| p = root / s["image"] | |
| if p not in img_cache: | |
| img_cache = {p: Image.open(p).convert("RGB")} | |
| t0 = time.perf_counter() | |
| h = model(img_cache[p], bbox=[float(v) for v in s["bbox"]], is_right=bool(s["is_right"])) | |
| out["time_ms"][i] = (time.perf_counter() - t0) * 1e3 | |
| out["rotmats"][i], out["betas"][i], out["cam"][i], out["cam_t_full"][i] = h.rotmats, h.betas, h.cam, h.cam_t_full | |
| if (i + 1) % 50 == 0 or i + 1 == n: | |
| el = time.time() - t_start | |
| print(f"[eval] {i + 1}/{n} ({el:.0f}s, {el / (i + 1):.2f}s/hand)", flush=True) | |
| model.close() | |
| np.savez(a.out, ids=np.array([s["id"] for s in samples]), sets=np.array([s["set"] for s in samples]), | |
| impl=a.impl, wall_s=time.time() - t_start, **out) | |
| print(f"[eval] wrote {a.out}: {n} hands in {time.time() - t_start:.0f}s") | |
| def pcc(u, v): | |
| u = u - u.mean() | |
| v = v - v.mean() | |
| return float(u @ v / max(np.linalg.norm(u) * np.linalg.norm(v), 1e-12)) | |
| def geodesic_deg(ra, rb): | |
| """Rotation angle between (..., 3, 3) rotation matrices ra and rb, degrees. | |
| Uses |ra - rb|_F = 2 sqrt(2) sin(theta / 2), which stays accurate for small angles (arccos of the trace does not: | |
| a 1e-6 trace error alone reads as about 0.08 degrees).""" | |
| f = np.linalg.norm(ra - rb, axis=(-2, -1)) | |
| return np.degrees(2.0 * np.arcsin(np.clip(f / (2.0 * np.sqrt(2.0)), 0.0, 1.0))) | |
| def stats(v): | |
| v = np.asarray(v, dtype=np.float64).reshape(-1) | |
| return {"mean": float(v.mean()), "median": float(np.median(v)), "p99": float(np.percentile(v, 99)), | |
| "max": float(v.max()), "min": float(v.min()), "n": int(v.size)} | |
| def compare(a): | |
| r, t = np.load(a.ref), np.load(a.test) | |
| if list(r["ids"]) != list(t["ids"]): | |
| # align on ids (the test run may cover a prefix) | |
| idx = {k: i for i, k in enumerate(r["ids"])} | |
| sel = [idx[k] for k in t["ids"]] | |
| else: | |
| sel = list(range(len(r["ids"]))) | |
| R = {k: r[k][sel] for k in ("rotmats", "betas", "cam", "cam_t_full")} | |
| T = {k: t[k] for k in ("rotmats", "betas", "cam", "cam_t_full")} | |
| sets = t["sets"] | |
| res = {"ref": str(a.ref), "test": str(a.test), "ref_impl": str(r["impl"]), "test_impl": str(t["impl"]), "sets": {}} | |
| for name in ["all"] + sorted(set(sets.tolist())): | |
| m = np.ones(len(sets), bool) if name == "all" else sets == name | |
| vr = np.concatenate([R["rotmats"][m].reshape(-1, 144), R["betas"][m], R["cam"][m]], 1) | |
| vt = np.concatenate([T["rotmats"][m].reshape(-1, 144), T["betas"][m], T["cam"][m]], 1) | |
| p = np.array([pcc(x, y) for x, y in zip(vt, vr)]) | |
| ang = geodesic_deg(T["rotmats"][m], R["rotmats"][m]) # (N, 16) | |
| dt = np.linalg.norm(T["cam_t_full"][m] - R["cam_t_full"][m], axis=1) * 1e3 | |
| dz = np.abs(T["cam_t_full"][m][:, 2] - R["cam_t_full"][m][:, 2]) / np.abs(R["cam_t_full"][m][:, 2]) * 100 | |
| ds = np.abs(T["cam"][m][:, 0] - R["cam"][m][:, 0]) / np.abs(R["cam"][m][:, 0]) * 100 | |
| res["sets"][name] = { | |
| "hands": int(m.sum()), | |
| "pcc157": {**stats(p), "below_0.9999": int((p < 0.9999).sum()), "below_0.999": int((p < 0.999).sum())}, | |
| "rot_err_deg_all16": stats(ang), | |
| "rot_err_deg_global": stats(ang[:, 0]), | |
| "rot_err_deg_hand_pose": stats(ang[:, 1:]), | |
| "rot_err_deg_per_hand_mean": stats(ang.mean(1)), | |
| "betas_abs_err": stats(np.abs(T["betas"][m] - R["betas"][m])), | |
| "cam_t_full_err_mm": stats(dt), | |
| "cam_t_full_depth_rel_err_pct": stats(dz), | |
| "cam_scale_rel_err_pct": stats(ds), | |
| "ref_cam_t_full_depth_m": stats(R["cam_t_full"][m][:, 2]), | |
| } | |
| s = res["sets"]["all"] | |
| print(f"[compare] {s['hands']} hands: PCC mean {s['pcc157']['mean']:.6f} min {s['pcc157']['min']:.6f}; " | |
| f"rotation error mean {s['rot_err_deg_all16']['mean']:.3f} deg max {s['rot_err_deg_all16']['max']:.3f}; " | |
| f"cam_t_full error mean {s['cam_t_full_err_mm']['mean']:.2f} mm max {s['cam_t_full_err_mm']['max']:.2f}") | |
| if a.json: | |
| Path(a.json).write_text(json.dumps(res, indent=1)) | |
| return res | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| sub = ap.add_subparsers(dest="cmd", required=True) | |
| r = sub.add_parser("run") | |
| r.add_argument("--impl", choices=["cpu", "tt", "http"], required=True) | |
| r.add_argument("--samples", required=True) | |
| r.add_argument("--root", required=True) | |
| r.add_argument("--out", required=True) | |
| r.add_argument("--url", default="http://127.0.0.1:20000") | |
| r.add_argument("--limit", type=int, default=0) | |
| r.add_argument("--threads", type=int, default=0, help="torch CPU threads (cpu impl)") | |
| c = sub.add_parser("compare") | |
| c.add_argument("--ref", required=True) | |
| c.add_argument("--test", required=True) | |
| c.add_argument("--json", default=None) | |
| a = ap.parse_args() | |
| if a.cmd == "run": | |
| run(a) | |
| else: | |
| compare(a) | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |