"""Turbo Vision miner for manak0/Detect-beverage-detect (cup / can / bottle). CPU-only ONNX inference with 2 intra-op threads (the compliance checker measures p95 latency on 2 vCPUs and re-runs the model to check that outputs are reproducible, so everything here is deterministic numpy / onnxruntime CPU). Repo layout: miner.py this file weights.onnx detector, input [1,3,S,S] RGB float 0..1, output [1, 4+3, N] (cx, cy, w, h in input pixels, then per-class sigmoid scores) config.json {"conf": [cup, can, bottle], "nms_iou": 0.5, "max_det": 100, "gate": 0.0, "rescue": [...], "esm": {"w": 3x3, "miss": 3x2, "draws": 256, "kmax": 24, "seed": 0}} Which boxes are sent: with an "esm" block in config.json, the boxes that maximise the EXPECTED validator score (class _Esm below; numbers fitted on our own model by tune_esm.py). Without it, per-class conf thresholds plus the first-box rescue. No `from __future__ import annotations` here: pydantic must see real types when the chute imports this file dynamically. """ import json from pathlib import Path import cv2 import numpy as np import onnxruntime as ort from numpy import ndarray from pydantic import BaseModel class BoundingBox(BaseModel): x1: int y1: int x2: int y2: int cls_id: int conf: float class TVFrameResult(BaseModel): frame_id: int boxes: list[BoundingBox] keypoints: list[tuple[int, int]] def _nms(boxes: ndarray, scores: ndarray, iou: float) -> list[int]: order = np.argsort(-scores, kind="stable") keep: list[int] = [] area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) while order.size: i = int(order[0]) keep.append(i) rest = order[1:] xx1 = np.maximum(boxes[i, 0], boxes[rest, 0]) yy1 = np.maximum(boxes[i, 1], boxes[rest, 1]) xx2 = np.minimum(boxes[i, 2], boxes[rest, 2]) yy2 = np.minimum(boxes[i, 3], boxes[rest, 3]) inter = np.maximum(0, xx2 - xx1) * np.maximum(0, yy2 - yy1) ov = inter / (area[i] + area[rest] - inter + 1e-7) order = rest[ov <= iou] return keep class _Esm: """Expected-score-maximising selection (same maths as esm_core.py, kept here so the repo is one file). The validator scores one image as max(0.6*map50 + 0.4*max(0, 1 - FP/10) - THETA, 0.01), map50 = mean AP over the classes that have ground truth. We do not know which candidates are right, so `draws` fixed scenarios are drawn: candidate j is right with probability P_j = sigmoid(w . [1, logit(conf), log10(area share) + 3]) and the class has Poisson(m0 + m1 * sum P) more objects that no candidate found. For "send the top k of each class" the formula is evaluated in every scenario and k is chosen by coordinate ascent on the average. The random numbers are fixed at start-up: the same image always gives the same boxes. """ THETA = 0.362454704407324514 def __init__(self, cfg: dict) -> None: self.w = np.array(cfg["w"], dtype=np.float64) self.miss = np.array(cfg["miss"], dtype=np.float64) self.draws = int(cfg.get("draws", 256)) self.kmax = int(cfg.get("kmax", 24)) rs = np.random.RandomState(int(cfg.get("seed", 0))) self.U = rs.random_sample((3, self.draws, self.kmax)) self.UM = np.stack([(rs.permutation(self.draws) + 0.5) / self.draws for _ in range(3)]) self.tri = np.triu(np.ones((self.kmax, self.kmax))) self.rank = np.arange(1, self.kmax + 1) self.logfact = np.concatenate([[0.0], np.cumsum(np.log(np.arange(1, 81)))]) self.m = np.arange(81) def prob(self, cls: ndarray, conf: ndarray, area: ndarray) -> ndarray: c = np.clip(np.asarray(conf, dtype=np.float64), 1e-4, 1 - 1e-4) a = np.log10(np.maximum(np.asarray(area, dtype=np.float64), 1e-6)) + 3.0 w = self.w[cls] return 1.0 / (1.0 + np.exp(-(w[:, 0] + w[:, 1] * np.log(c / (1 - c)) + w[:, 2] * a))) def select(self, plist: list) -> list: """plist: per class, P(right) of its candidates in sending order (at most kmax). Returns k per class.""" S = self.draws ctps, aps, pres = [], [], np.zeros((S, 3), dtype=bool) for c, p in enumerate(plist): K = len(p) t = self.U[c, :, :K] < p[None, :] lam = max(float(self.miss[c, 0] + self.miss[c, 1] * p.sum()), 1e-3) cdf = np.cumsum(np.exp(-lam + self.m * np.log(lam) - self.logfact)) n = t.sum(1) + np.minimum(np.searchsorted(cdf, self.UM[c], side="left"), 80) ctp = np.zeros((S, K + 1)) ap = np.zeros((S, K + 1)) if K: tf = t.astype(np.float64) cum = np.cumsum(tf, 1) ctp[:, 1:] = cum prec = cum / self.rank[:K] env = np.maximum.accumulate(prec[:, None, :] * self.tri[:K, :K], axis=2) # AP precision envelope ap[:, 1:] = np.einsum("sj,sjk->sk", tf, env) / np.maximum(n, 1)[:, None] ctps.append(ctp) aps.append(ap) pres[:, c] = n > 0 npres = pres.sum(1) den = np.maximum(npres, 1) k = [int((p >= 0.5).sum()) for p in plist] for _ in range(4): changed = False for c in range(3): oth_ap = np.zeros(S) oth_fp = np.zeros(S) for d in range(3): if d != c: oth_ap += pres[:, d] * aps[d][:, k[d]] oth_fp += k[d] - ctps[d][:, k[d]] K = aps[c].shape[1] - 1 apm = (oth_ap[:, None] + pres[:, c:c + 1] * aps[c]) / den[:, None] fp = oth_fp[:, None] + np.arange(K + 1)[None, :] - ctps[c] fps = np.maximum(0.0, 1.0 - fp / 10.0) tot = np.where(npres[:, None] > 0, 0.6 * apm + 0.4 * fps, fps) ev = np.maximum(tot - self.THETA, 0.01).mean(0) b = int(np.argmax(ev)) if ev[b] > ev[k[c]] + 1e-12: k[c] = b changed = True if not changed: break return k class Miner: def __init__(self, path_hf_repo: Path) -> None: path_hf_repo = Path(path_hf_repo) cfg_path = path_hf_repo / "config.json" self.cfg = json.loads(cfg_path.read_text()) if cfg_path.exists() else {} self.conf = np.array(self.cfg.get("conf", [0.4, 0.4, 0.4]), dtype=np.float32) self.nms_iou = float(self.cfg.get("nms_iou", 0.5)) self.max_det = int(self.cfg.get("max_det", 100)) self.gate = float(self.cfg.get("gate", 0.0)) # per class: if no box passes conf, keep the class's best candidate when it scores >= rescue # (1.0 = off). map50 averages over the classes in an image, so the first box of a class is worth most. self.rescue = np.array(self.cfg.get("rescue", [1.0, 1.0, 1.0]), dtype=np.float32) self.esm = _Esm(self.cfg["esm"]) if self.cfg.get("esm") else None so = ort.SessionOptions() so.intra_op_num_threads = 2 so.inter_op_num_threads = 1 so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL self.session = ort.InferenceSession(str(path_hf_repo / "weights.onnx"), so, providers=["CPUExecutionProvider"]) self.input_name = self.session.get_inputs()[0].name self.size = int(self.session.get_inputs()[0].shape[-1]) self.buf = np.empty((1, 3, self.size, self.size), dtype=np.float32) cv2.setNumThreads(1) warm = np.zeros((1024, 1024, 3), dtype=np.uint8) for _ in range(2): self._raw(warm) def __repr__(self) -> str: return f"ONNX P3 detector {self.size}px conf={self.conf.tolist()}" def _raw(self, image: ndarray): """letterbox -> ONNX -> (boxes xyxy in original pixels, per-class scores [N,3])""" h, w = image.shape[:2] s = self.size r = min(s / h, s / w) nw, nh = int(round(w * r)), int(round(h * r)) top, left = (s - nh) // 2, (s - nw) // 2 img = cv2.resize(image, (nw, nh), interpolation=cv2.INTER_LINEAR) if (nw, nh) != (w, h) else image self.buf.fill(114 / 255) k = np.float32(1 / 255) for c in range(3): # BGR -> RGB, HWC -> CHW, 0..1 np.multiply(img[:, :, 2 - c], k, out=self.buf[0, c, top:top + nh, left:left + nw], dtype=np.float32) out = self.session.run(None, {self.input_name: self.buf})[0][0].T # [N, 4+nc] xy, wh, sc = out[:, :2], out[:, 2:4], out[:, 4:] boxes = np.concatenate([xy - wh / 2, xy + wh / 2], 1) boxes[:, [0, 2]] = (boxes[:, [0, 2]] - left) / r boxes[:, [1, 3]] = (boxes[:, [1, 3]] - top) / r boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, w - 1) boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, h - 1) return boxes, sc def candidates(self, image: ndarray, floor: float = 0.05): """All per-class NMS survivors above `floor` (used for threshold tuning).""" boxes, sc = self._raw(image) cls = sc.argmax(1) conf = sc[np.arange(len(sc)), cls] m = conf >= floor boxes, conf, cls = boxes[m], conf[m], cls[m] keep: list[int] = [] for c in range(sc.shape[1]): idx = np.flatnonzero(cls == c) if idx.size: keep.extend(idx[_nms(boxes[idx], conf[idx], self.nms_iou)].tolist()) keep = sorted(keep, key=lambda i: -conf[i]) return boxes[keep], conf[keep], cls[keep] def _detect_esm(self, image: ndarray) -> list[BoundingBox]: boxes, conf, cls = self.candidates(image, floor=0.05) q = np.round(boxes).astype(int) ok = (q[:, 2] > q[:, 0]) & (q[:, 3] > q[:, 1]) q, conf, cls = q[ok], conf[ok].astype(np.float64), cls[ok] if not len(q): return [] h, w = image.shape[:2] p = self.esm.prob(cls, conf, (q[:, 2] - q[:, 0]) * (q[:, 3] - q[:, 1]) / float(h * w)) idxs = [np.flatnonzero(cls == c)[: self.esm.kmax] for c in range(3)] # candidates are sorted by score k = self.esm.select([p[i] for i in idxs]) out = [(q[i], c, conf[i]) for c in range(3) for i in idxs[c][: k[c]]][: self.max_det] return [BoundingBox(x1=int(a[0]), y1=int(a[1]), x2=int(a[2]), y2=int(a[3]), cls_id=int(c), conf=float(f)) for a, c, f in out] def _detect(self, image: ndarray) -> list[BoundingBox]: if self.esm is not None: return self._detect_esm(image) boxes, conf, cls = self.candidates(image, floor=float(min(self.conf.min(), self.rescue.min(), 0.05))) if conf.size == 0 or float(conf.max()) < self.gate: return [] m = conf >= self.conf[cls] for c in range(len(self.rescue)): if self.rescue[c] < 1.0 and not m[cls == c].any(): idx = np.flatnonzero((cls == c) & (conf >= self.rescue[c])) if idx.size: m[idx[0]] = True # candidates are sorted by score, so idx[0] is the class's best boxes, conf, cls = boxes[m][: self.max_det], conf[m][: self.max_det], cls[m][: self.max_det] q = np.round(boxes).astype(int) ok = (q[:, 2] > q[:, 0]) & (q[:, 3] > q[:, 1]) return [BoundingBox(x1=int(a[0]), y1=int(a[1]), x2=int(a[2]), y2=int(a[3]), cls_id=int(c), conf=float(f)) for a, c, f in zip(q[ok], cls[ok], conf[ok])] def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]: results = [] for i, im in enumerate(batch_images): try: boxes = self._detect(im) except Exception as e: # one bad frame must not fail the batch print(f"frame {offset + i} failed: {e}") boxes = [] results.append(TVFrameResult(frame_id=offset + i, boxes=boxes, keypoints=[(0, 0)] * n_keypoints)) return results