File size: 12,114 Bytes
cf9704d 951ce90 cf9704d 951ce90 cf9704d 951ce90 cf9704d 951ce90 cf9704d 951ce90 cf9704d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 | """Turbo Vision miner for manak0/Detect-beverage-detect (cup / can / bottle).
CPU-only ONNX inference with 2 intra-op threads (the compliance checker measures
p95 latency on 2 vCPUs and re-runs the model to check that outputs are reproducible,
so everything here is deterministic numpy / onnxruntime CPU).
Repo layout:
miner.py this file
weights.onnx detector, input [1,3,S,S] RGB float 0..1, output [1, 4+3, N]
(cx, cy, w, h in input pixels, then per-class sigmoid scores)
config.json {"conf": [cup, can, bottle], "nms_iou": 0.5, "max_det": 100, "gate": 0.0, "rescue": [...],
"esm": {"w": 3x3, "miss": 3x2, "draws": 256, "kmax": 24, "seed": 0}}
Which boxes are sent: with an "esm" block in config.json, the boxes that maximise the EXPECTED validator score
(class _Esm below; numbers fitted on our own model by tune_esm.py). Without it, per-class conf thresholds plus
the first-box rescue.
No `from __future__ import annotations` here: pydantic must see real types when the
chute imports this file dynamically.
"""
import json
from pathlib import Path
import cv2
import numpy as np
import onnxruntime as ort
from numpy import ndarray
from pydantic import BaseModel
class BoundingBox(BaseModel):
x1: int
y1: int
x2: int
y2: int
cls_id: int
conf: float
class TVFrameResult(BaseModel):
frame_id: int
boxes: list[BoundingBox]
keypoints: list[tuple[int, int]]
def _nms(boxes: ndarray, scores: ndarray, iou: float) -> list[int]:
order = np.argsort(-scores, kind="stable")
keep: list[int] = []
area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
while order.size:
i = int(order[0])
keep.append(i)
rest = order[1:]
xx1 = np.maximum(boxes[i, 0], boxes[rest, 0])
yy1 = np.maximum(boxes[i, 1], boxes[rest, 1])
xx2 = np.minimum(boxes[i, 2], boxes[rest, 2])
yy2 = np.minimum(boxes[i, 3], boxes[rest, 3])
inter = np.maximum(0, xx2 - xx1) * np.maximum(0, yy2 - yy1)
ov = inter / (area[i] + area[rest] - inter + 1e-7)
order = rest[ov <= iou]
return keep
class _Esm:
"""Expected-score-maximising selection (same maths as esm_core.py, kept here so the repo is one file).
The validator scores one image as max(0.6*map50 + 0.4*max(0, 1 - FP/10) - THETA, 0.01), map50 = mean AP over
the classes that have ground truth. We do not know which candidates are right, so `draws` fixed scenarios are
drawn: candidate j is right with probability P_j = sigmoid(w . [1, logit(conf), log10(area share) + 3]) and
the class has Poisson(m0 + m1 * sum P) more objects that no candidate found. For "send the top k of each
class" the formula is evaluated in every scenario and k is chosen by coordinate ascent on the average.
The random numbers are fixed at start-up: the same image always gives the same boxes.
"""
THETA = 0.362454704407324514
def __init__(self, cfg: dict) -> None:
self.w = np.array(cfg["w"], dtype=np.float64)
self.miss = np.array(cfg["miss"], dtype=np.float64)
self.draws = int(cfg.get("draws", 256))
self.kmax = int(cfg.get("kmax", 24))
rs = np.random.RandomState(int(cfg.get("seed", 0)))
self.U = rs.random_sample((3, self.draws, self.kmax))
self.UM = np.stack([(rs.permutation(self.draws) + 0.5) / self.draws for _ in range(3)])
self.tri = np.triu(np.ones((self.kmax, self.kmax)))
self.rank = np.arange(1, self.kmax + 1)
self.logfact = np.concatenate([[0.0], np.cumsum(np.log(np.arange(1, 81)))])
self.m = np.arange(81)
def prob(self, cls: ndarray, conf: ndarray, area: ndarray) -> ndarray:
c = np.clip(np.asarray(conf, dtype=np.float64), 1e-4, 1 - 1e-4)
a = np.log10(np.maximum(np.asarray(area, dtype=np.float64), 1e-6)) + 3.0
w = self.w[cls]
return 1.0 / (1.0 + np.exp(-(w[:, 0] + w[:, 1] * np.log(c / (1 - c)) + w[:, 2] * a)))
def select(self, plist: list) -> list:
"""plist: per class, P(right) of its candidates in sending order (at most kmax). Returns k per class."""
S = self.draws
ctps, aps, pres = [], [], np.zeros((S, 3), dtype=bool)
for c, p in enumerate(plist):
K = len(p)
t = self.U[c, :, :K] < p[None, :]
lam = max(float(self.miss[c, 0] + self.miss[c, 1] * p.sum()), 1e-3)
cdf = np.cumsum(np.exp(-lam + self.m * np.log(lam) - self.logfact))
n = t.sum(1) + np.minimum(np.searchsorted(cdf, self.UM[c], side="left"), 80)
ctp = np.zeros((S, K + 1))
ap = np.zeros((S, K + 1))
if K:
tf = t.astype(np.float64)
cum = np.cumsum(tf, 1)
ctp[:, 1:] = cum
prec = cum / self.rank[:K]
env = np.maximum.accumulate(prec[:, None, :] * self.tri[:K, :K], axis=2) # AP precision envelope
ap[:, 1:] = np.einsum("sj,sjk->sk", tf, env) / np.maximum(n, 1)[:, None]
ctps.append(ctp)
aps.append(ap)
pres[:, c] = n > 0
npres = pres.sum(1)
den = np.maximum(npres, 1)
k = [int((p >= 0.5).sum()) for p in plist]
for _ in range(4):
changed = False
for c in range(3):
oth_ap = np.zeros(S)
oth_fp = np.zeros(S)
for d in range(3):
if d != c:
oth_ap += pres[:, d] * aps[d][:, k[d]]
oth_fp += k[d] - ctps[d][:, k[d]]
K = aps[c].shape[1] - 1
apm = (oth_ap[:, None] + pres[:, c:c + 1] * aps[c]) / den[:, None]
fp = oth_fp[:, None] + np.arange(K + 1)[None, :] - ctps[c]
fps = np.maximum(0.0, 1.0 - fp / 10.0)
tot = np.where(npres[:, None] > 0, 0.6 * apm + 0.4 * fps, fps)
ev = np.maximum(tot - self.THETA, 0.01).mean(0)
b = int(np.argmax(ev))
if ev[b] > ev[k[c]] + 1e-12:
k[c] = b
changed = True
if not changed:
break
return k
class Miner:
def __init__(self, path_hf_repo: Path) -> None:
path_hf_repo = Path(path_hf_repo)
cfg_path = path_hf_repo / "config.json"
self.cfg = json.loads(cfg_path.read_text()) if cfg_path.exists() else {}
self.conf = np.array(self.cfg.get("conf", [0.4, 0.4, 0.4]), dtype=np.float32)
self.nms_iou = float(self.cfg.get("nms_iou", 0.5))
self.max_det = int(self.cfg.get("max_det", 100))
self.gate = float(self.cfg.get("gate", 0.0))
# per class: if no box passes conf, keep the class's best candidate when it scores >= rescue
# (1.0 = off). map50 averages over the classes in an image, so the first box of a class is worth most.
self.rescue = np.array(self.cfg.get("rescue", [1.0, 1.0, 1.0]), dtype=np.float32)
self.esm = _Esm(self.cfg["esm"]) if self.cfg.get("esm") else None
so = ort.SessionOptions()
so.intra_op_num_threads = 2
so.inter_op_num_threads = 1
so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
self.session = ort.InferenceSession(str(path_hf_repo / "weights.onnx"), so, providers=["CPUExecutionProvider"])
self.input_name = self.session.get_inputs()[0].name
self.size = int(self.session.get_inputs()[0].shape[-1])
self.buf = np.empty((1, 3, self.size, self.size), dtype=np.float32)
cv2.setNumThreads(1)
warm = np.zeros((1024, 1024, 3), dtype=np.uint8)
for _ in range(2):
self._raw(warm)
def __repr__(self) -> str:
return f"ONNX P3 detector {self.size}px conf={self.conf.tolist()}"
def _raw(self, image: ndarray):
"""letterbox -> ONNX -> (boxes xyxy in original pixels, per-class scores [N,3])"""
h, w = image.shape[:2]
s = self.size
r = min(s / h, s / w)
nw, nh = int(round(w * r)), int(round(h * r))
top, left = (s - nh) // 2, (s - nw) // 2
img = cv2.resize(image, (nw, nh), interpolation=cv2.INTER_LINEAR) if (nw, nh) != (w, h) else image
self.buf.fill(114 / 255)
k = np.float32(1 / 255)
for c in range(3): # BGR -> RGB, HWC -> CHW, 0..1
np.multiply(img[:, :, 2 - c], k, out=self.buf[0, c, top:top + nh, left:left + nw], dtype=np.float32)
out = self.session.run(None, {self.input_name: self.buf})[0][0].T # [N, 4+nc]
xy, wh, sc = out[:, :2], out[:, 2:4], out[:, 4:]
boxes = np.concatenate([xy - wh / 2, xy + wh / 2], 1)
boxes[:, [0, 2]] = (boxes[:, [0, 2]] - left) / r
boxes[:, [1, 3]] = (boxes[:, [1, 3]] - top) / r
boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, w - 1)
boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, h - 1)
return boxes, sc
def candidates(self, image: ndarray, floor: float = 0.05):
"""All per-class NMS survivors above `floor` (used for threshold tuning)."""
boxes, sc = self._raw(image)
cls = sc.argmax(1)
conf = sc[np.arange(len(sc)), cls]
m = conf >= floor
boxes, conf, cls = boxes[m], conf[m], cls[m]
keep: list[int] = []
for c in range(sc.shape[1]):
idx = np.flatnonzero(cls == c)
if idx.size:
keep.extend(idx[_nms(boxes[idx], conf[idx], self.nms_iou)].tolist())
keep = sorted(keep, key=lambda i: -conf[i])
return boxes[keep], conf[keep], cls[keep]
def _detect_esm(self, image: ndarray) -> list[BoundingBox]:
boxes, conf, cls = self.candidates(image, floor=0.05)
q = np.round(boxes).astype(int)
ok = (q[:, 2] > q[:, 0]) & (q[:, 3] > q[:, 1])
q, conf, cls = q[ok], conf[ok].astype(np.float64), cls[ok]
if not len(q):
return []
h, w = image.shape[:2]
p = self.esm.prob(cls, conf, (q[:, 2] - q[:, 0]) * (q[:, 3] - q[:, 1]) / float(h * w))
idxs = [np.flatnonzero(cls == c)[: self.esm.kmax] for c in range(3)] # candidates are sorted by score
k = self.esm.select([p[i] for i in idxs])
out = [(q[i], c, conf[i]) for c in range(3) for i in idxs[c][: k[c]]][: self.max_det]
return [BoundingBox(x1=int(a[0]), y1=int(a[1]), x2=int(a[2]), y2=int(a[3]), cls_id=int(c), conf=float(f))
for a, c, f in out]
def _detect(self, image: ndarray) -> list[BoundingBox]:
if self.esm is not None:
return self._detect_esm(image)
boxes, conf, cls = self.candidates(image, floor=float(min(self.conf.min(), self.rescue.min(), 0.05)))
if conf.size == 0 or float(conf.max()) < self.gate:
return []
m = conf >= self.conf[cls]
for c in range(len(self.rescue)):
if self.rescue[c] < 1.0 and not m[cls == c].any():
idx = np.flatnonzero((cls == c) & (conf >= self.rescue[c]))
if idx.size:
m[idx[0]] = True # candidates are sorted by score, so idx[0] is the class's best
boxes, conf, cls = boxes[m][: self.max_det], conf[m][: self.max_det], cls[m][: self.max_det]
q = np.round(boxes).astype(int)
ok = (q[:, 2] > q[:, 0]) & (q[:, 3] > q[:, 1])
return [BoundingBox(x1=int(a[0]), y1=int(a[1]), x2=int(a[2]), y2=int(a[3]), cls_id=int(c), conf=float(f))
for a, c, f in zip(q[ok], cls[ok], conf[ok])]
def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]:
results = []
for i, im in enumerate(batch_images):
try:
boxes = self._detect(im)
except Exception as e: # one bad frame must not fail the batch
print(f"frame {offset + i} failed: {e}")
boxes = []
results.append(TVFrameResult(frame_id=offset + i, boxes=boxes, keypoints=[(0, 0)] * n_keypoints))
return results
|