ScoreVision / miner.py
BashirEmran's picture
scorevision: push artifact
951ce90 verified
Raw History Blame
12.1 kB
"""Turbo Vision miner for manak0/Detect-beverage-detect (cup / can / bottle).
CPU-only ONNX inference with 2 intra-op threads (the compliance checker measures
p95 latency on 2 vCPUs and re-runs the model to check that outputs are reproducible,
so everything here is deterministic numpy / onnxruntime CPU).
Repo layout:
miner.py this file
weights.onnx detector, input [1,3,S,S] RGB float 0..1, output [1, 4+3, N]
(cx, cy, w, h in input pixels, then per-class sigmoid scores)
config.json {"conf": [cup, can, bottle], "nms_iou": 0.5, "max_det": 100, "gate": 0.0, "rescue": [...],
"esm": {"w": 3x3, "miss": 3x2, "draws": 256, "kmax": 24, "seed": 0}}
Which boxes are sent: with an "esm" block in config.json, the boxes that maximise the EXPECTED validator score
(class _Esm below; numbers fitted on our own model by tune_esm.py). Without it, per-class conf thresholds plus
the first-box rescue.
No `from __future__ import annotations` here: pydantic must see real types when the
chute imports this file dynamically.
"""
import json
from pathlib import Path
import cv2
import numpy as np
import onnxruntime as ort
from numpy import ndarray
from pydantic import BaseModel
class BoundingBox(BaseModel):
x1: int
y1: int
x2: int
y2: int
cls_id: int
conf: float
class TVFrameResult(BaseModel):
frame_id: int
boxes: list[BoundingBox]
keypoints: list[tuple[int, int]]
def _nms(boxes: ndarray, scores: ndarray, iou: float) -> list[int]:
order = np.argsort(-scores, kind="stable")
keep: list[int] = []
area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
while order.size:
i = int(order[0])
keep.append(i)
rest = order[1:]
xx1 = np.maximum(boxes[i, 0], boxes[rest, 0])
yy1 = np.maximum(boxes[i, 1], boxes[rest, 1])
xx2 = np.minimum(boxes[i, 2], boxes[rest, 2])
yy2 = np.minimum(boxes[i, 3], boxes[rest, 3])
inter = np.maximum(0, xx2 - xx1) * np.maximum(0, yy2 - yy1)
ov = inter / (area[i] + area[rest] - inter + 1e-7)
order = rest[ov <= iou]
return keep
class _Esm:
"""Expected-score-maximising selection (same maths as esm_core.py, kept here so the repo is one file).
The validator scores one image as max(0.6*map50 + 0.4*max(0, 1 - FP/10) - THETA, 0.01), map50 = mean AP over
the classes that have ground truth. We do not know which candidates are right, so `draws` fixed scenarios are
drawn: candidate j is right with probability P_j = sigmoid(w . [1, logit(conf), log10(area share) + 3]) and
the class has Poisson(m0 + m1 * sum P) more objects that no candidate found. For "send the top k of each
class" the formula is evaluated in every scenario and k is chosen by coordinate ascent on the average.
The random numbers are fixed at start-up: the same image always gives the same boxes.
"""
THETA = 0.362454704407324514
def __init__(self, cfg: dict) -> None:
self.w = np.array(cfg["w"], dtype=np.float64)
self.miss = np.array(cfg["miss"], dtype=np.float64)
self.draws = int(cfg.get("draws", 256))
self.kmax = int(cfg.get("kmax", 24))
rs = np.random.RandomState(int(cfg.get("seed", 0)))
self.U = rs.random_sample((3, self.draws, self.kmax))
self.UM = np.stack([(rs.permutation(self.draws) + 0.5) / self.draws for _ in range(3)])
self.tri = np.triu(np.ones((self.kmax, self.kmax)))
self.rank = np.arange(1, self.kmax + 1)
self.logfact = np.concatenate([[0.0], np.cumsum(np.log(np.arange(1, 81)))])
self.m = np.arange(81)
def prob(self, cls: ndarray, conf: ndarray, area: ndarray) -> ndarray:
c = np.clip(np.asarray(conf, dtype=np.float64), 1e-4, 1 - 1e-4)
a = np.log10(np.maximum(np.asarray(area, dtype=np.float64), 1e-6)) + 3.0
w = self.w[cls]
return 1.0 / (1.0 + np.exp(-(w[:, 0] + w[:, 1] * np.log(c / (1 - c)) + w[:, 2] * a)))
def select(self, plist: list) -> list:
"""plist: per class, P(right) of its candidates in sending order (at most kmax). Returns k per class."""
S = self.draws
ctps, aps, pres = [], [], np.zeros((S, 3), dtype=bool)
for c, p in enumerate(plist):
K = len(p)
t = self.U[c, :, :K] < p[None, :]
lam = max(float(self.miss[c, 0] + self.miss[c, 1] * p.sum()), 1e-3)
cdf = np.cumsum(np.exp(-lam + self.m * np.log(lam) - self.logfact))
n = t.sum(1) + np.minimum(np.searchsorted(cdf, self.UM[c], side="left"), 80)
ctp = np.zeros((S, K + 1))
ap = np.zeros((S, K + 1))
if K:
tf = t.astype(np.float64)
cum = np.cumsum(tf, 1)
ctp[:, 1:] = cum
prec = cum / self.rank[:K]
env = np.maximum.accumulate(prec[:, None, :] * self.tri[:K, :K], axis=2) # AP precision envelope
ap[:, 1:] = np.einsum("sj,sjk->sk", tf, env) / np.maximum(n, 1)[:, None]
ctps.append(ctp)
aps.append(ap)
pres[:, c] = n > 0
npres = pres.sum(1)
den = np.maximum(npres, 1)
k = [int((p >= 0.5).sum()) for p in plist]
for _ in range(4):
changed = False
for c in range(3):
oth_ap = np.zeros(S)
oth_fp = np.zeros(S)
for d in range(3):
if d != c:
oth_ap += pres[:, d] * aps[d][:, k[d]]
oth_fp += k[d] - ctps[d][:, k[d]]
K = aps[c].shape[1] - 1
apm = (oth_ap[:, None] + pres[:, c:c + 1] * aps[c]) / den[:, None]
fp = oth_fp[:, None] + np.arange(K + 1)[None, :] - ctps[c]
fps = np.maximum(0.0, 1.0 - fp / 10.0)
tot = np.where(npres[:, None] > 0, 0.6 * apm + 0.4 * fps, fps)
ev = np.maximum(tot - self.THETA, 0.01).mean(0)
b = int(np.argmax(ev))
if ev[b] > ev[k[c]] + 1e-12:
k[c] = b
changed = True
if not changed:
break
return k
class Miner:
def __init__(self, path_hf_repo: Path) -> None:
path_hf_repo = Path(path_hf_repo)
cfg_path = path_hf_repo / "config.json"
self.cfg = json.loads(cfg_path.read_text()) if cfg_path.exists() else {}
self.conf = np.array(self.cfg.get("conf", [0.4, 0.4, 0.4]), dtype=np.float32)
self.nms_iou = float(self.cfg.get("nms_iou", 0.5))
self.max_det = int(self.cfg.get("max_det", 100))
self.gate = float(self.cfg.get("gate", 0.0))
# per class: if no box passes conf, keep the class's best candidate when it scores >= rescue
# (1.0 = off). map50 averages over the classes in an image, so the first box of a class is worth most.
self.rescue = np.array(self.cfg.get("rescue", [1.0, 1.0, 1.0]), dtype=np.float32)
self.esm = _Esm(self.cfg["esm"]) if self.cfg.get("esm") else None
so = ort.SessionOptions()
so.intra_op_num_threads = 2
so.inter_op_num_threads = 1
so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
self.session = ort.InferenceSession(str(path_hf_repo / "weights.onnx"), so, providers=["CPUExecutionProvider"])
self.input_name = self.session.get_inputs()[0].name
self.size = int(self.session.get_inputs()[0].shape[-1])
self.buf = np.empty((1, 3, self.size, self.size), dtype=np.float32)
cv2.setNumThreads(1)
warm = np.zeros((1024, 1024, 3), dtype=np.uint8)
for _ in range(2):
self._raw(warm)
def __repr__(self) -> str:
return f"ONNX P3 detector {self.size}px conf={self.conf.tolist()}"
def _raw(self, image: ndarray):
"""letterbox -> ONNX -> (boxes xyxy in original pixels, per-class scores [N,3])"""
h, w = image.shape[:2]
s = self.size
r = min(s / h, s / w)
nw, nh = int(round(w * r)), int(round(h * r))
top, left = (s - nh) // 2, (s - nw) // 2
img = cv2.resize(image, (nw, nh), interpolation=cv2.INTER_LINEAR) if (nw, nh) != (w, h) else image
self.buf.fill(114 / 255)
k = np.float32(1 / 255)
for c in range(3): # BGR -> RGB, HWC -> CHW, 0..1
np.multiply(img[:, :, 2 - c], k, out=self.buf[0, c, top:top + nh, left:left + nw], dtype=np.float32)
out = self.session.run(None, {self.input_name: self.buf})[0][0].T # [N, 4+nc]
xy, wh, sc = out[:, :2], out[:, 2:4], out[:, 4:]
boxes = np.concatenate([xy - wh / 2, xy + wh / 2], 1)
boxes[:, [0, 2]] = (boxes[:, [0, 2]] - left) / r
boxes[:, [1, 3]] = (boxes[:, [1, 3]] - top) / r
boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, w - 1)
boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, h - 1)
return boxes, sc
def candidates(self, image: ndarray, floor: float = 0.05):
"""All per-class NMS survivors above `floor` (used for threshold tuning)."""
boxes, sc = self._raw(image)
cls = sc.argmax(1)
conf = sc[np.arange(len(sc)), cls]
m = conf >= floor
boxes, conf, cls = boxes[m], conf[m], cls[m]
keep: list[int] = []
for c in range(sc.shape[1]):
idx = np.flatnonzero(cls == c)
if idx.size:
keep.extend(idx[_nms(boxes[idx], conf[idx], self.nms_iou)].tolist())
keep = sorted(keep, key=lambda i: -conf[i])
return boxes[keep], conf[keep], cls[keep]
def _detect_esm(self, image: ndarray) -> list[BoundingBox]:
boxes, conf, cls = self.candidates(image, floor=0.05)
q = np.round(boxes).astype(int)
ok = (q[:, 2] > q[:, 0]) & (q[:, 3] > q[:, 1])
q, conf, cls = q[ok], conf[ok].astype(np.float64), cls[ok]
if not len(q):
return []
h, w = image.shape[:2]
p = self.esm.prob(cls, conf, (q[:, 2] - q[:, 0]) * (q[:, 3] - q[:, 1]) / float(h * w))
idxs = [np.flatnonzero(cls == c)[: self.esm.kmax] for c in range(3)] # candidates are sorted by score
k = self.esm.select([p[i] for i in idxs])
out = [(q[i], c, conf[i]) for c in range(3) for i in idxs[c][: k[c]]][: self.max_det]
return [BoundingBox(x1=int(a[0]), y1=int(a[1]), x2=int(a[2]), y2=int(a[3]), cls_id=int(c), conf=float(f))
for a, c, f in out]
def _detect(self, image: ndarray) -> list[BoundingBox]:
if self.esm is not None:
return self._detect_esm(image)
boxes, conf, cls = self.candidates(image, floor=float(min(self.conf.min(), self.rescue.min(), 0.05)))
if conf.size == 0 or float(conf.max()) < self.gate:
return []
m = conf >= self.conf[cls]
for c in range(len(self.rescue)):
if self.rescue[c] < 1.0 and not m[cls == c].any():
idx = np.flatnonzero((cls == c) & (conf >= self.rescue[c]))
if idx.size:
m[idx[0]] = True # candidates are sorted by score, so idx[0] is the class's best
boxes, conf, cls = boxes[m][: self.max_det], conf[m][: self.max_det], cls[m][: self.max_det]
q = np.round(boxes).astype(int)
ok = (q[:, 2] > q[:, 0]) & (q[:, 3] > q[:, 1])
return [BoundingBox(x1=int(a[0]), y1=int(a[1]), x2=int(a[2]), y2=int(a[3]), cls_id=int(c), conf=float(f))
for a, c, f in zip(q[ok], cls[ok], conf[ok])]
def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]:
results = []
for i, im in enumerate(batch_images):
try:
boxes = self._detect(im)
except Exception as e: # one bad frame must not fail the batch
print(f"frame {offset + i} failed: {e}")
boxes = []
results.append(TVFrameResult(frame_id=offset + i, boxes=boxes, keypoints=[(0, 0)] * n_keypoints))
return results