import math from pathlib import Path import cv2 import numpy as np import onnxruntime as ort from numpy import ndarray from pydantic import BaseModel # candidates: image gate, confidence floor, per-class NMS, cross-class dedup CONF = np.array((0.425, 0.52, 0.40), dtype=np.float32) FLOOR = 0.2 GATE = 0.6 NMS_IOU = 0.45 DEDUP_IOU = 0.9 PRE_TOPK = 1000 MAX_DET = 300 # selection: per class keep the top-k by conf, k chosen to maximise a Monte Carlo estimate of the expected validator score NMAX = 32 NS = 256 SEED = 1234567 THETA = 0.3624547044073245 SHIFT = -1.0 # logit offset added to every P(TP) # P(TP) = sigmoid(a + b*logit(conf) + c*log10(area/1024^2) + d*logit*log10area), per class cup/can/bottle CAL = np.array([[-1.985495, 0.07277, -1.153825, -0.585014], [-0.705046, 3.412035, -0.543796, 0.383031], [-1.778816, 2.69706, -1.218205, 0.197397]]) # objects the candidates miss ~ Poisson(a + b * sum P(TP)), per class MISS = np.array([[1.942812, 0.229558], [0.552509, 0.193124], [1.384334, 0.53347]]) class BoundingBox(BaseModel): x1: int y1: int x2: int y2: int cls_id: int conf: float class TVFrameResult(BaseModel): frame_id: int boxes: list[BoundingBox] keypoints: list[tuple[int, int]] def _iou(b: ndarray) -> ndarray: area = np.maximum(0, b[:, 2] - b[:, 0]) * np.maximum(0, b[:, 3] - b[:, 1]) iw = np.maximum(0, np.minimum(b[:, None, 2], b[None, :, 2]) - np.maximum(b[:, None, 0], b[None, :, 0])) ih = np.maximum(0, np.minimum(b[:, None, 3], b[None, :, 3]) - np.maximum(b[:, None, 1], b[None, :, 1])) inter = iw * ih return inter / (area[:, None] + area[None, :] - inter + 1e-7) def _greedy(over: ndarray, order: ndarray) -> list[int]: gone = np.zeros(len(order), bool) keep = [] for i in order: if not gone[i]: keep.append(int(i)) gone |= over[i] return keep def _nms(b: ndarray, s: ndarray, iou: float) -> list[int]: order = np.argsort(-s, kind="stable") over = _iou(b) > iou if not over.sum() > len(b): # only the diagonal overlaps: nothing to suppress return order.tolist() return _greedy(over, order) def _splitmix(seed: int, n: int) -> ndarray: z = np.arange(1, n + 1, dtype=np.uint64) * np.uint64(0x9E3779B97F4A7C15) + np.uint64(seed) z = (z ^ (z >> np.uint64(30))) * np.uint64(0xBF58476D1CE4E5B9) z = (z ^ (z >> np.uint64(27))) * np.uint64(0x94D049BB133111EB) z = z ^ (z >> np.uint64(31)) return (z >> np.uint64(11)).astype(np.float64) / 2.0 ** 53 def _stratified(seed: int, shape: tuple[int, ...]) -> ndarray: """Latin-hypercube uniforms along the last axis: one draw per stratum.""" n = math.prod(shape) perm = np.argsort(_splitmix(seed, n).reshape(shape), axis=-1, kind="stable") return (perm + _splitmix(seed + 1, n).reshape(shape)) / shape[-1] _LGF = np.array([math.lgamma(k + 1) for k in range(64)]) def _poisson(lam: float, u: ndarray) -> ndarray: k = np.arange(64) cdf = np.cumsum(np.exp(-lam + k * math.log(max(lam, 1e-12)) - _LGF)) return np.minimum(np.searchsorted(cdf, u), 63) def _ap_paths(t: ndarray) -> ndarray: """Sum of precision at each TP after keeping the top 0..n boxes, per Monte Carlo draw.""" m, n = t.shape out = np.zeros((m, n + 1)) if n: out[:, 1:] = np.cumsum(t * (np.cumsum(t, 1) / np.arange(1, n + 1)), 1) return out class Miner: def __init__(self, path_hf_repo: Path) -> None: so = ort.SessionOptions() so.intra_op_num_threads = 2 so.inter_op_num_threads = 1 so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL so.add_session_config_entry("session.intra_op.allow_spinning", "1") so.add_session_config_entry("session.inter_op.allow_spinning", "1") self.session = ort.InferenceSession(str(Path(path_hf_repo) / "weights.onnx"), so, providers=["CPUExecutionProvider"]) self.input_name = self.session.get_inputs()[0].name self.size = int(self.session.get_inputs()[0].shape[-1]) self.buf = np.full((1, 3, self.size, self.size), 114 / 255, dtype=np.float32) self.u = np.ascontiguousarray(_stratified(SEED, (3, NMAX, NS)).transpose(0, 2, 1)) # [class, draw, rank] self.v = _stratified(SEED + 7919, (3, NS)) # [class, draw] cv2.setNumThreads(1) def __repr__(self) -> str: return f"ONNX {self.size}" def _infer(self, image: ndarray) -> tuple[ndarray, tuple[float, float, float, int, int]]: h, w = image.shape[:2] s = self.size scale = min(s / h, s / w) nw, nh = round(w * scale), round(h * scale) dx, dy = (s - nw) / 2, (s - nh) / 2 top, left = round(dy - 0.1), round(dx - 0.1) r = cv2.resize(image, (nw, nh)) if (nw, nh) != (w, h) else image inp = self.buf if nw != s or nh != s: inp.fill(114 / 255) k = np.float32(1 / 255) for c in range(3): # BGR → RGB, HWC → CHW, /255, straight into the reused input buffer np.multiply(r[:, :, 2 - c], k, out=inp[0, c, top:top + nh, left:left + nw], dtype=np.float32) return self.session.run(None, {self.input_name: inp})[0][0].T, (scale, dx, dy, w, h) def _candidates(self, raw: ndarray, meta: tuple[float, float, float, int, int]): scale, dx, dy, w, h = meta conf = raw[:, 4:].max(1) cls = raw[:, 4:].argmax(1) if not len(conf) or float(conf.max()) < GATE: return None keep = conf >= FLOOR raw, conf, cls = raw[keep], conf[keep], cls[keep] if not len(raw): return None b = np.stack([raw[:, 0] - raw[:, 2] / 2, raw[:, 1] - raw[:, 3] / 2, raw[:, 0] + raw[:, 2] / 2, raw[:, 1] + raw[:, 3] / 2], 1) b[:, [0, 2]] = np.clip((b[:, [0, 2]] - dx) / scale, 0, w - 1) b[:, [1, 3]] = np.clip((b[:, [1, 3]] - dy) / scale, 0, h - 1) bw, bh = np.maximum(0, b[:, 2] - b[:, 0]), np.maximum(0, b[:, 3] - b[:, 1]) ar = np.maximum(bw / np.maximum(bh, 1e-6), bh / np.maximum(bw, 1e-6)) ok = (bw >= 3) & (bh >= 3) & (bw * bh >= 8) & (bw * bh <= 0.95 * w * h) & (ar <= 12) b, conf, cls = b[ok], conf[ok], cls[ok] if not len(conf): return None if len(conf) > PRE_TOPK: t = np.argpartition(-conf, PRE_TOPK)[:PRE_TOPK] b, conf, cls = b[t], conf[t], cls[t] sel = sorted(j for c in range(len(CONF)) for j in np.flatnonzero(cls == c)[_nms(b[cls == c], conf[cls == c], NMS_IOU)]) b, conf, cls = b[sel], conf[sel], cls[sel] if len(conf) > MAX_DET: t = np.argsort(-conf)[:MAX_DET] b, conf, cls = b[t], conf[t], cls[t] if len(b) > 1: iw = np.maximum(0, np.minimum(b[:, None, 2], b[None, :, 2]) - np.maximum(b[:, None, 0], b[None, :, 0])) ih = np.maximum(0, np.minimum(b[:, None, 3], b[None, :, 3]) - np.maximum(b[:, None, 1], b[None, :, 1])) a_ = np.maximum(0, b[:, 2] - b[:, 0]) * np.maximum(0, b[:, 3] - b[:, 1]) inside = iw * ih / (np.minimum(a_[:, None], a_[None, :]) + 1e-7) # share of the smaller box inside the other over = (_iou(b) > DEDUP_IOU) | (inside >= 0.85) np.fill_diagonal(over, False) area = (b[:, 2] - b[:, 0]) * (b[:, 3] - b[:, 1]) order = np.lexsort((-area, -(conf - CONF[cls]))) kept = sorted(_greedy(over, order)) if over.any() else list(range(len(b))) b, conf, cls = b[kept], conf[kept], cls[kept] q = np.stack([np.floor(b[:, 0]), np.floor(b[:, 1]), np.ceil(b[:, 2]), np.ceil(b[:, 3])], 1) q[:, [0, 2]] = np.clip(q[:, [0, 2]], 0, w) q[:, [1, 3]] = np.clip(q[:, [1, 3]], 0, h) ok = (q[:, 2] > q[:, 0]) & (q[:, 3] > q[:, 1]) return q[ok].astype(int), conf[ok], cls[ok] def _choose(self, q: ndarray, conf: ndarray, cls: ndarray) -> ndarray: ap, fp, ngt, idx = [], [], [], [] for c in range(3): i = np.flatnonzero(cls == c) i = i[np.argsort(-conf[i], kind="stable")][:NMAX] idx.append(i) s = np.clip(conf[i].astype(np.float64), 1e-4, 1 - 1e-4) lg = np.log(s / (1 - s)) la = np.log10(np.clip((q[i, 2] - q[i, 0]) * (q[i, 3] - q[i, 1]) / (1024 * 1024), 1e-6, 1)) p = 1 / (1 + np.exp(-(CAL[c, 0] + CAL[c, 1] * lg + CAL[c, 2] * la + CAL[c, 3] * lg * la + SHIFT))) t = self.u[c][:, :len(i)] < p[None, :] # [draw, rank]: is the box a TP in this draw ap.append(_ap_paths(t)) fp.append(np.arange(len(i) + 1)[None, :] - np.concatenate([np.zeros((NS, 1)), np.cumsum(t, 1)], 1)) ngt.append(t.sum(1) + _poisson(max(MISS[c, 0] + MISS[c, 1] * float(p.sum()), 0.0), self.v[c])) ngt = np.stack(ngt, 1) present = ngt > 0 n_present = np.maximum(present.sum(1), 1) empty = present.sum(1) == 0 part = [np.where(present[:, c, None], ap[c] / np.maximum(ngt[:, c, None], 1), 0.0) for c in range(3)] k = [0, 0, 0] a_tot = sum(part[c][:, 0] for c in range(3)) f_tot = sum(fp[c][:, 0] for c in range(3)) def value(c: int) -> ndarray: # expected max(score - THETA, .01) for every k of class c, others fixed a = (a_tot - part[c][:, k[c]])[:, None] + part[c] f = (f_tot - fp[c][:, k[c]])[:, None] + fp[c] n = sum(k) - k[c] + np.arange(len(idx[c]) + 1) pos = np.maximum(0.6 * a / n_present[:, None] + 0.4 * np.maximum(0.0, 1 - f / 10) - THETA, 0.01) neg = np.maximum(np.maximum(0.0, 1 - n / 10) - THETA, 0.01) return np.where(empty[:, None], neg[None, :], pos).mean(0) best = value(0)[0] for _ in range(3): # coordinate ascent over the per-class counts moved = False for c in range(3): v = value(c) j = int(np.argmax(v)) if v[j] > best + 1e-9: a_tot += part[c][:, j] - part[c][:, k[c]] f_tot += fp[c][:, j] - fp[c][:, k[c]] best, k[c], moved = float(v[j]), j, True if not moved: break return np.concatenate([idx[c][:k[c]] for c in range(3)]).astype(int) def _select(self, raw: ndarray, meta: tuple[float, float, float, int, int]) -> list[BoundingBox]: got = self._candidates(raw, meta) if got is None: return [] q, conf, cls = got sel = self._choose(q, conf, cls) return [BoundingBox(x1=int(q[i, 0]), y1=int(q[i, 1]), x2=int(q[i, 2]), y2=int(q[i, 3]), cls_id=int(cls[i]), conf=float(min(1.0, max(0.0, conf[i])))) for i in sorted(sel.tolist(), key=lambda i: -conf[i])] def _detect(self, image: ndarray) -> list[BoundingBox]: try: return self._select(*self._infer(image)) except Exception as e: # one bad frame must not fail the whole batch print(f"frame failed: {e}") return [] def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]: return [TVFrameResult(frame_id=offset + i, boxes=self._detect(im), keypoints=[(0, 0)] * n_keypoints) for i, im in enumerate(batch_images)]