ScoreVision / miner.py
BashirEmran's picture
scorevision: push artifact
dd8896d verified
Raw History Blame Contribute Delete
11.4 kB
import math
from pathlib import Path
import cv2
import numpy as np
import onnxruntime as ort
from numpy import ndarray
from pydantic import BaseModel
# candidates: image gate, confidence floor, per-class NMS, cross-class dedup
CONF = np.array((0.425, 0.52, 0.40), dtype=np.float32)
FLOOR = 0.2
GATE = 0.6
NMS_IOU = 0.45
DEDUP_IOU = 0.9
PRE_TOPK = 1000
MAX_DET = 300
# selection: per class keep the top-k by conf, k chosen to maximise a Monte Carlo estimate of the expected validator score
NMAX = 32
NS = 256
SEED = 1234567
THETA = 0.3624547044073245
SHIFT = -1.0 # logit offset added to every P(TP)
# P(TP) = sigmoid(a + b*logit(conf) + c*log10(area/1024^2) + d*logit*log10area), per class cup/can/bottle
CAL = np.array([[-1.985495, 0.07277, -1.153825, -0.585014], [-0.705046, 3.412035, -0.543796, 0.383031], [-1.778816, 2.69706, -1.218205, 0.197397]])
# objects the candidates miss ~ Poisson(a + b * sum P(TP)), per class
MISS = np.array([[1.942812, 0.229558], [0.552509, 0.193124], [1.384334, 0.53347]])
class BoundingBox(BaseModel):
x1: int
y1: int
x2: int
y2: int
cls_id: int
conf: float
class TVFrameResult(BaseModel):
frame_id: int
boxes: list[BoundingBox]
keypoints: list[tuple[int, int]]
def _iou(b: ndarray) -> ndarray:
area = np.maximum(0, b[:, 2] - b[:, 0]) * np.maximum(0, b[:, 3] - b[:, 1])
iw = np.maximum(0, np.minimum(b[:, None, 2], b[None, :, 2]) - np.maximum(b[:, None, 0], b[None, :, 0]))
ih = np.maximum(0, np.minimum(b[:, None, 3], b[None, :, 3]) - np.maximum(b[:, None, 1], b[None, :, 1]))
inter = iw * ih
return inter / (area[:, None] + area[None, :] - inter + 1e-7)
def _greedy(over: ndarray, order: ndarray) -> list[int]:
gone = np.zeros(len(order), bool)
keep = []
for i in order:
if not gone[i]:
keep.append(int(i))
gone |= over[i]
return keep
def _nms(b: ndarray, s: ndarray, iou: float) -> list[int]:
order = np.argsort(-s, kind="stable")
over = _iou(b) > iou
if not over.sum() > len(b): # only the diagonal overlaps: nothing to suppress
return order.tolist()
return _greedy(over, order)
def _splitmix(seed: int, n: int) -> ndarray:
z = np.arange(1, n + 1, dtype=np.uint64) * np.uint64(0x9E3779B97F4A7C15) + np.uint64(seed)
z = (z ^ (z >> np.uint64(30))) * np.uint64(0xBF58476D1CE4E5B9)
z = (z ^ (z >> np.uint64(27))) * np.uint64(0x94D049BB133111EB)
z = z ^ (z >> np.uint64(31))
return (z >> np.uint64(11)).astype(np.float64) / 2.0 ** 53
def _stratified(seed: int, shape: tuple[int, ...]) -> ndarray:
"""Latin-hypercube uniforms along the last axis: one draw per stratum."""
n = math.prod(shape)
perm = np.argsort(_splitmix(seed, n).reshape(shape), axis=-1, kind="stable")
return (perm + _splitmix(seed + 1, n).reshape(shape)) / shape[-1]
_LGF = np.array([math.lgamma(k + 1) for k in range(64)])
def _poisson(lam: float, u: ndarray) -> ndarray:
k = np.arange(64)
cdf = np.cumsum(np.exp(-lam + k * math.log(max(lam, 1e-12)) - _LGF))
return np.minimum(np.searchsorted(cdf, u), 63)
def _ap_paths(t: ndarray) -> ndarray:
"""Sum of precision at each TP after keeping the top 0..n boxes, per Monte Carlo draw."""
m, n = t.shape
out = np.zeros((m, n + 1))
if n:
out[:, 1:] = np.cumsum(t * (np.cumsum(t, 1) / np.arange(1, n + 1)), 1)
return out
class Miner:
def __init__(self, path_hf_repo: Path) -> None:
so = ort.SessionOptions()
so.intra_op_num_threads = 2
so.inter_op_num_threads = 1
so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
so.add_session_config_entry("session.intra_op.allow_spinning", "1")
so.add_session_config_entry("session.inter_op.allow_spinning", "1")
self.session = ort.InferenceSession(str(Path(path_hf_repo) / "weights.onnx"), so, providers=["CPUExecutionProvider"])
self.input_name = self.session.get_inputs()[0].name
self.size = int(self.session.get_inputs()[0].shape[-1])
self.buf = np.full((1, 3, self.size, self.size), 114 / 255, dtype=np.float32)
self.u = np.ascontiguousarray(_stratified(SEED, (3, NMAX, NS)).transpose(0, 2, 1)) # [class, draw, rank]
self.v = _stratified(SEED + 7919, (3, NS)) # [class, draw]
cv2.setNumThreads(1)
def __repr__(self) -> str:
return f"ONNX {self.size}"
def _infer(self, image: ndarray) -> tuple[ndarray, tuple[float, float, float, int, int]]:
h, w = image.shape[:2]
s = self.size
scale = min(s / h, s / w)
nw, nh = round(w * scale), round(h * scale)
dx, dy = (s - nw) / 2, (s - nh) / 2
top, left = round(dy - 0.1), round(dx - 0.1)
r = cv2.resize(image, (nw, nh)) if (nw, nh) != (w, h) else image
inp = self.buf
if nw != s or nh != s:
inp.fill(114 / 255)
k = np.float32(1 / 255)
for c in range(3): # BGR → RGB, HWC → CHW, /255, straight into the reused input buffer
np.multiply(r[:, :, 2 - c], k, out=inp[0, c, top:top + nh, left:left + nw], dtype=np.float32)
return self.session.run(None, {self.input_name: inp})[0][0].T, (scale, dx, dy, w, h)
def _candidates(self, raw: ndarray, meta: tuple[float, float, float, int, int]):
scale, dx, dy, w, h = meta
conf = raw[:, 4:].max(1)
cls = raw[:, 4:].argmax(1)
if not len(conf) or float(conf.max()) < GATE:
return None
keep = conf >= FLOOR
raw, conf, cls = raw[keep], conf[keep], cls[keep]
if not len(raw):
return None
b = np.stack([raw[:, 0] - raw[:, 2] / 2, raw[:, 1] - raw[:, 3] / 2, raw[:, 0] + raw[:, 2] / 2, raw[:, 1] + raw[:, 3] / 2], 1)
b[:, [0, 2]] = np.clip((b[:, [0, 2]] - dx) / scale, 0, w - 1)
b[:, [1, 3]] = np.clip((b[:, [1, 3]] - dy) / scale, 0, h - 1)
bw, bh = np.maximum(0, b[:, 2] - b[:, 0]), np.maximum(0, b[:, 3] - b[:, 1])
ar = np.maximum(bw / np.maximum(bh, 1e-6), bh / np.maximum(bw, 1e-6))
ok = (bw >= 3) & (bh >= 3) & (bw * bh >= 8) & (bw * bh <= 0.95 * w * h) & (ar <= 12)
b, conf, cls = b[ok], conf[ok], cls[ok]
if not len(conf):
return None
if len(conf) > PRE_TOPK:
t = np.argpartition(-conf, PRE_TOPK)[:PRE_TOPK]
b, conf, cls = b[t], conf[t], cls[t]
sel = sorted(j for c in range(len(CONF)) for j in np.flatnonzero(cls == c)[_nms(b[cls == c], conf[cls == c], NMS_IOU)])
b, conf, cls = b[sel], conf[sel], cls[sel]
if len(conf) > MAX_DET:
t = np.argsort(-conf)[:MAX_DET]
b, conf, cls = b[t], conf[t], cls[t]
if len(b) > 1:
iw = np.maximum(0, np.minimum(b[:, None, 2], b[None, :, 2]) - np.maximum(b[:, None, 0], b[None, :, 0]))
ih = np.maximum(0, np.minimum(b[:, None, 3], b[None, :, 3]) - np.maximum(b[:, None, 1], b[None, :, 1]))
a_ = np.maximum(0, b[:, 2] - b[:, 0]) * np.maximum(0, b[:, 3] - b[:, 1])
inside = iw * ih / (np.minimum(a_[:, None], a_[None, :]) + 1e-7) # share of the smaller box inside the other
over = (_iou(b) > DEDUP_IOU) | (inside >= 0.85)
np.fill_diagonal(over, False)
area = (b[:, 2] - b[:, 0]) * (b[:, 3] - b[:, 1])
order = np.lexsort((-area, -(conf - CONF[cls])))
kept = sorted(_greedy(over, order)) if over.any() else list(range(len(b)))
b, conf, cls = b[kept], conf[kept], cls[kept]
q = np.stack([np.floor(b[:, 0]), np.floor(b[:, 1]), np.ceil(b[:, 2]), np.ceil(b[:, 3])], 1)
q[:, [0, 2]] = np.clip(q[:, [0, 2]], 0, w)
q[:, [1, 3]] = np.clip(q[:, [1, 3]], 0, h)
ok = (q[:, 2] > q[:, 0]) & (q[:, 3] > q[:, 1])
return q[ok].astype(int), conf[ok], cls[ok]
def _choose(self, q: ndarray, conf: ndarray, cls: ndarray) -> ndarray:
ap, fp, ngt, idx = [], [], [], []
for c in range(3):
i = np.flatnonzero(cls == c)
i = i[np.argsort(-conf[i], kind="stable")][:NMAX]
idx.append(i)
s = np.clip(conf[i].astype(np.float64), 1e-4, 1 - 1e-4)
lg = np.log(s / (1 - s))
la = np.log10(np.clip((q[i, 2] - q[i, 0]) * (q[i, 3] - q[i, 1]) / (1024 * 1024), 1e-6, 1))
p = 1 / (1 + np.exp(-(CAL[c, 0] + CAL[c, 1] * lg + CAL[c, 2] * la + CAL[c, 3] * lg * la + SHIFT)))
t = self.u[c][:, :len(i)] < p[None, :] # [draw, rank]: is the box a TP in this draw
ap.append(_ap_paths(t))
fp.append(np.arange(len(i) + 1)[None, :] - np.concatenate([np.zeros((NS, 1)), np.cumsum(t, 1)], 1))
ngt.append(t.sum(1) + _poisson(max(MISS[c, 0] + MISS[c, 1] * float(p.sum()), 0.0), self.v[c]))
ngt = np.stack(ngt, 1)
present = ngt > 0
n_present = np.maximum(present.sum(1), 1)
empty = present.sum(1) == 0
part = [np.where(present[:, c, None], ap[c] / np.maximum(ngt[:, c, None], 1), 0.0) for c in range(3)]
k = [0, 0, 0]
a_tot = sum(part[c][:, 0] for c in range(3))
f_tot = sum(fp[c][:, 0] for c in range(3))
def value(c: int) -> ndarray: # expected max(score - THETA, .01) for every k of class c, others fixed
a = (a_tot - part[c][:, k[c]])[:, None] + part[c]
f = (f_tot - fp[c][:, k[c]])[:, None] + fp[c]
n = sum(k) - k[c] + np.arange(len(idx[c]) + 1)
pos = np.maximum(0.6 * a / n_present[:, None] + 0.4 * np.maximum(0.0, 1 - f / 10) - THETA, 0.01)
neg = np.maximum(np.maximum(0.0, 1 - n / 10) - THETA, 0.01)
return np.where(empty[:, None], neg[None, :], pos).mean(0)
best = value(0)[0]
for _ in range(3): # coordinate ascent over the per-class counts
moved = False
for c in range(3):
v = value(c)
j = int(np.argmax(v))
if v[j] > best + 1e-9:
a_tot += part[c][:, j] - part[c][:, k[c]]
f_tot += fp[c][:, j] - fp[c][:, k[c]]
best, k[c], moved = float(v[j]), j, True
if not moved:
break
return np.concatenate([idx[c][:k[c]] for c in range(3)]).astype(int)
def _select(self, raw: ndarray, meta: tuple[float, float, float, int, int]) -> list[BoundingBox]:
got = self._candidates(raw, meta)
if got is None:
return []
q, conf, cls = got
sel = self._choose(q, conf, cls)
return [BoundingBox(x1=int(q[i, 0]), y1=int(q[i, 1]), x2=int(q[i, 2]), y2=int(q[i, 3]), cls_id=int(cls[i]),
conf=float(min(1.0, max(0.0, conf[i])))) for i in sorted(sel.tolist(), key=lambda i: -conf[i])]
def _detect(self, image: ndarray) -> list[BoundingBox]:
try:
return self._select(*self._infer(image))
except Exception as e: # one bad frame must not fail the whole batch
print(f"frame failed: {e}")
return []
def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]:
return [TVFrameResult(frame_id=offset + i, boxes=self._detect(im), keypoints=[(0, 0)] * n_keypoints)
for i, im in enumerate(batch_images)]