File size: 12,114 Bytes
cf9704d
 
 
 
 
 
 
 
 
 
951ce90
 
 
 
 
 
cf9704d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
951ce90
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
cf9704d
 
 
 
 
 
 
 
 
 
 
 
951ce90
cf9704d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
951ce90
 
 
 
 
 
 
 
 
 
 
 
 
 
 
cf9704d
951ce90
 
cf9704d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
"""Turbo Vision miner for manak0/Detect-beverage-detect (cup / can / bottle).

CPU-only ONNX inference with 2 intra-op threads (the compliance checker measures
p95 latency on 2 vCPUs and re-runs the model to check that outputs are reproducible,
so everything here is deterministic numpy / onnxruntime CPU).

Repo layout:
  miner.py       this file
  weights.onnx   detector, input [1,3,S,S] RGB float 0..1, output [1, 4+3, N]
                 (cx, cy, w, h in input pixels, then per-class sigmoid scores)
  config.json    {"conf": [cup, can, bottle], "nms_iou": 0.5, "max_det": 100, "gate": 0.0, "rescue": [...],
                  "esm": {"w": 3x3, "miss": 3x2, "draws": 256, "kmax": 24, "seed": 0}}

Which boxes are sent: with an "esm" block in config.json, the boxes that maximise the EXPECTED validator score
(class _Esm below; numbers fitted on our own model by tune_esm.py). Without it, per-class conf thresholds plus
the first-box rescue.

No `from __future__ import annotations` here: pydantic must see real types when the
chute imports this file dynamically.
"""
import json
from pathlib import Path

import cv2
import numpy as np
import onnxruntime as ort
from numpy import ndarray
from pydantic import BaseModel


class BoundingBox(BaseModel):
    x1: int
    y1: int
    x2: int
    y2: int
    cls_id: int
    conf: float


class TVFrameResult(BaseModel):
    frame_id: int
    boxes: list[BoundingBox]
    keypoints: list[tuple[int, int]]


def _nms(boxes: ndarray, scores: ndarray, iou: float) -> list[int]:
    order = np.argsort(-scores, kind="stable")
    keep: list[int] = []
    area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
    while order.size:
        i = int(order[0])
        keep.append(i)
        rest = order[1:]
        xx1 = np.maximum(boxes[i, 0], boxes[rest, 0])
        yy1 = np.maximum(boxes[i, 1], boxes[rest, 1])
        xx2 = np.minimum(boxes[i, 2], boxes[rest, 2])
        yy2 = np.minimum(boxes[i, 3], boxes[rest, 3])
        inter = np.maximum(0, xx2 - xx1) * np.maximum(0, yy2 - yy1)
        ov = inter / (area[i] + area[rest] - inter + 1e-7)
        order = rest[ov <= iou]
    return keep


class _Esm:
    """Expected-score-maximising selection (same maths as esm_core.py, kept here so the repo is one file).

    The validator scores one image as max(0.6*map50 + 0.4*max(0, 1 - FP/10) - THETA, 0.01), map50 = mean AP over
    the classes that have ground truth. We do not know which candidates are right, so `draws` fixed scenarios are
    drawn: candidate j is right with probability P_j = sigmoid(w . [1, logit(conf), log10(area share) + 3]) and
    the class has Poisson(m0 + m1 * sum P) more objects that no candidate found. For "send the top k of each
    class" the formula is evaluated in every scenario and k is chosen by coordinate ascent on the average.
    The random numbers are fixed at start-up: the same image always gives the same boxes.
    """
    THETA = 0.362454704407324514

    def __init__(self, cfg: dict) -> None:
        self.w = np.array(cfg["w"], dtype=np.float64)
        self.miss = np.array(cfg["miss"], dtype=np.float64)
        self.draws = int(cfg.get("draws", 256))
        self.kmax = int(cfg.get("kmax", 24))
        rs = np.random.RandomState(int(cfg.get("seed", 0)))
        self.U = rs.random_sample((3, self.draws, self.kmax))
        self.UM = np.stack([(rs.permutation(self.draws) + 0.5) / self.draws for _ in range(3)])
        self.tri = np.triu(np.ones((self.kmax, self.kmax)))
        self.rank = np.arange(1, self.kmax + 1)
        self.logfact = np.concatenate([[0.0], np.cumsum(np.log(np.arange(1, 81)))])
        self.m = np.arange(81)

    def prob(self, cls: ndarray, conf: ndarray, area: ndarray) -> ndarray:
        c = np.clip(np.asarray(conf, dtype=np.float64), 1e-4, 1 - 1e-4)
        a = np.log10(np.maximum(np.asarray(area, dtype=np.float64), 1e-6)) + 3.0
        w = self.w[cls]
        return 1.0 / (1.0 + np.exp(-(w[:, 0] + w[:, 1] * np.log(c / (1 - c)) + w[:, 2] * a)))

    def select(self, plist: list) -> list:
        """plist: per class, P(right) of its candidates in sending order (at most kmax). Returns k per class."""
        S = self.draws
        ctps, aps, pres = [], [], np.zeros((S, 3), dtype=bool)
        for c, p in enumerate(plist):
            K = len(p)
            t = self.U[c, :, :K] < p[None, :]
            lam = max(float(self.miss[c, 0] + self.miss[c, 1] * p.sum()), 1e-3)
            cdf = np.cumsum(np.exp(-lam + self.m * np.log(lam) - self.logfact))
            n = t.sum(1) + np.minimum(np.searchsorted(cdf, self.UM[c], side="left"), 80)
            ctp = np.zeros((S, K + 1))
            ap = np.zeros((S, K + 1))
            if K:
                tf = t.astype(np.float64)
                cum = np.cumsum(tf, 1)
                ctp[:, 1:] = cum
                prec = cum / self.rank[:K]
                env = np.maximum.accumulate(prec[:, None, :] * self.tri[:K, :K], axis=2)  # AP precision envelope
                ap[:, 1:] = np.einsum("sj,sjk->sk", tf, env) / np.maximum(n, 1)[:, None]
            ctps.append(ctp)
            aps.append(ap)
            pres[:, c] = n > 0
        npres = pres.sum(1)
        den = np.maximum(npres, 1)
        k = [int((p >= 0.5).sum()) for p in plist]
        for _ in range(4):
            changed = False
            for c in range(3):
                oth_ap = np.zeros(S)
                oth_fp = np.zeros(S)
                for d in range(3):
                    if d != c:
                        oth_ap += pres[:, d] * aps[d][:, k[d]]
                        oth_fp += k[d] - ctps[d][:, k[d]]
                K = aps[c].shape[1] - 1
                apm = (oth_ap[:, None] + pres[:, c:c + 1] * aps[c]) / den[:, None]
                fp = oth_fp[:, None] + np.arange(K + 1)[None, :] - ctps[c]
                fps = np.maximum(0.0, 1.0 - fp / 10.0)
                tot = np.where(npres[:, None] > 0, 0.6 * apm + 0.4 * fps, fps)
                ev = np.maximum(tot - self.THETA, 0.01).mean(0)
                b = int(np.argmax(ev))
                if ev[b] > ev[k[c]] + 1e-12:
                    k[c] = b
                    changed = True
            if not changed:
                break
        return k


class Miner:
    def __init__(self, path_hf_repo: Path) -> None:
        path_hf_repo = Path(path_hf_repo)
        cfg_path = path_hf_repo / "config.json"
        self.cfg = json.loads(cfg_path.read_text()) if cfg_path.exists() else {}
        self.conf = np.array(self.cfg.get("conf", [0.4, 0.4, 0.4]), dtype=np.float32)
        self.nms_iou = float(self.cfg.get("nms_iou", 0.5))
        self.max_det = int(self.cfg.get("max_det", 100))
        self.gate = float(self.cfg.get("gate", 0.0))
        # per class: if no box passes conf, keep the class's best candidate when it scores >= rescue
        # (1.0 = off). map50 averages over the classes in an image, so the first box of a class is worth most.
        self.rescue = np.array(self.cfg.get("rescue", [1.0, 1.0, 1.0]), dtype=np.float32)
        self.esm = _Esm(self.cfg["esm"]) if self.cfg.get("esm") else None
        so = ort.SessionOptions()
        so.intra_op_num_threads = 2
        so.inter_op_num_threads = 1
        so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
        so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
        self.session = ort.InferenceSession(str(path_hf_repo / "weights.onnx"), so, providers=["CPUExecutionProvider"])
        self.input_name = self.session.get_inputs()[0].name
        self.size = int(self.session.get_inputs()[0].shape[-1])
        self.buf = np.empty((1, 3, self.size, self.size), dtype=np.float32)
        cv2.setNumThreads(1)
        warm = np.zeros((1024, 1024, 3), dtype=np.uint8)
        for _ in range(2):
            self._raw(warm)

    def __repr__(self) -> str:
        return f"ONNX P3 detector {self.size}px conf={self.conf.tolist()}"

    def _raw(self, image: ndarray):
        """letterbox -> ONNX -> (boxes xyxy in original pixels, per-class scores [N,3])"""
        h, w = image.shape[:2]
        s = self.size
        r = min(s / h, s / w)
        nw, nh = int(round(w * r)), int(round(h * r))
        top, left = (s - nh) // 2, (s - nw) // 2
        img = cv2.resize(image, (nw, nh), interpolation=cv2.INTER_LINEAR) if (nw, nh) != (w, h) else image
        self.buf.fill(114 / 255)
        k = np.float32(1 / 255)
        for c in range(3):  # BGR -> RGB, HWC -> CHW, 0..1
            np.multiply(img[:, :, 2 - c], k, out=self.buf[0, c, top:top + nh, left:left + nw], dtype=np.float32)
        out = self.session.run(None, {self.input_name: self.buf})[0][0].T  # [N, 4+nc]
        xy, wh, sc = out[:, :2], out[:, 2:4], out[:, 4:]
        boxes = np.concatenate([xy - wh / 2, xy + wh / 2], 1)
        boxes[:, [0, 2]] = (boxes[:, [0, 2]] - left) / r
        boxes[:, [1, 3]] = (boxes[:, [1, 3]] - top) / r
        boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, w - 1)
        boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, h - 1)
        return boxes, sc

    def candidates(self, image: ndarray, floor: float = 0.05):
        """All per-class NMS survivors above `floor` (used for threshold tuning)."""
        boxes, sc = self._raw(image)
        cls = sc.argmax(1)
        conf = sc[np.arange(len(sc)), cls]
        m = conf >= floor
        boxes, conf, cls = boxes[m], conf[m], cls[m]
        keep: list[int] = []
        for c in range(sc.shape[1]):
            idx = np.flatnonzero(cls == c)
            if idx.size:
                keep.extend(idx[_nms(boxes[idx], conf[idx], self.nms_iou)].tolist())
        keep = sorted(keep, key=lambda i: -conf[i])
        return boxes[keep], conf[keep], cls[keep]

    def _detect_esm(self, image: ndarray) -> list[BoundingBox]:
        boxes, conf, cls = self.candidates(image, floor=0.05)
        q = np.round(boxes).astype(int)
        ok = (q[:, 2] > q[:, 0]) & (q[:, 3] > q[:, 1])
        q, conf, cls = q[ok], conf[ok].astype(np.float64), cls[ok]
        if not len(q):
            return []
        h, w = image.shape[:2]
        p = self.esm.prob(cls, conf, (q[:, 2] - q[:, 0]) * (q[:, 3] - q[:, 1]) / float(h * w))
        idxs = [np.flatnonzero(cls == c)[: self.esm.kmax] for c in range(3)]  # candidates are sorted by score
        k = self.esm.select([p[i] for i in idxs])
        out = [(q[i], c, conf[i]) for c in range(3) for i in idxs[c][: k[c]]][: self.max_det]
        return [BoundingBox(x1=int(a[0]), y1=int(a[1]), x2=int(a[2]), y2=int(a[3]), cls_id=int(c), conf=float(f))
                for a, c, f in out]

    def _detect(self, image: ndarray) -> list[BoundingBox]:
        if self.esm is not None:
            return self._detect_esm(image)
        boxes, conf, cls = self.candidates(image, floor=float(min(self.conf.min(), self.rescue.min(), 0.05)))
        if conf.size == 0 or float(conf.max()) < self.gate:
            return []
        m = conf >= self.conf[cls]
        for c in range(len(self.rescue)):
            if self.rescue[c] < 1.0 and not m[cls == c].any():
                idx = np.flatnonzero((cls == c) & (conf >= self.rescue[c]))
                if idx.size:
                    m[idx[0]] = True  # candidates are sorted by score, so idx[0] is the class's best
        boxes, conf, cls = boxes[m][: self.max_det], conf[m][: self.max_det], cls[m][: self.max_det]
        q = np.round(boxes).astype(int)
        ok = (q[:, 2] > q[:, 0]) & (q[:, 3] > q[:, 1])
        return [BoundingBox(x1=int(a[0]), y1=int(a[1]), x2=int(a[2]), y2=int(a[3]), cls_id=int(c), conf=float(f))
                for a, c, f in zip(q[ok], cls[ok], conf[ok])]

    def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]:
        results = []
        for i, im in enumerate(batch_images):
            try:
                boxes = self._detect(im)
            except Exception as e:  # one bad frame must not fail the batch
                print(f"frame {offset + i} failed: {e}")
                boxes = []
            results.append(TVFrameResult(frame_id=offset + i, boxes=boxes, keypoints=[(0, 0)] * n_keypoints))
        return results