File size: 11,391 Bytes
dd8896d
cf9704d
 
 
 
 
 
 
 
dd8896d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
cf9704d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dd8896d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
cf9704d
 
 
dd8896d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
951ce90
 
cf9704d
 
 
 
 
 
 
dd8896d
 
 
cf9704d
 
dd8896d
 
 
cf9704d
 
 
dd8896d
cf9704d
dd8896d
cf9704d
 
dd8896d
 
 
 
 
 
 
 
cf9704d
dd8896d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
951ce90
dd8896d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
951ce90
cf9704d
dd8896d
 
 
 
cf9704d
 
 
dd8896d
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
import math
from pathlib import Path

import cv2
import numpy as np
import onnxruntime as ort
from numpy import ndarray
from pydantic import BaseModel

# candidates: image gate, confidence floor, per-class NMS, cross-class dedup
CONF = np.array((0.425, 0.52, 0.40), dtype=np.float32)
FLOOR = 0.2
GATE = 0.6
NMS_IOU = 0.45
DEDUP_IOU = 0.9
PRE_TOPK = 1000
MAX_DET = 300
# selection: per class keep the top-k by conf, k chosen to maximise a Monte Carlo estimate of the expected validator score
NMAX = 32
NS = 256
SEED = 1234567
THETA = 0.3624547044073245
SHIFT = -1.0  # logit offset added to every P(TP)
# P(TP) = sigmoid(a + b*logit(conf) + c*log10(area/1024^2) + d*logit*log10area), per class cup/can/bottle
CAL = np.array([[-1.985495, 0.07277, -1.153825, -0.585014], [-0.705046, 3.412035, -0.543796, 0.383031], [-1.778816, 2.69706, -1.218205, 0.197397]])
# objects the candidates miss ~ Poisson(a + b * sum P(TP)), per class
MISS = np.array([[1.942812, 0.229558], [0.552509, 0.193124], [1.384334, 0.53347]])


class BoundingBox(BaseModel):
    x1: int
    y1: int
    x2: int
    y2: int
    cls_id: int
    conf: float


class TVFrameResult(BaseModel):
    frame_id: int
    boxes: list[BoundingBox]
    keypoints: list[tuple[int, int]]


def _iou(b: ndarray) -> ndarray:
    area = np.maximum(0, b[:, 2] - b[:, 0]) * np.maximum(0, b[:, 3] - b[:, 1])
    iw = np.maximum(0, np.minimum(b[:, None, 2], b[None, :, 2]) - np.maximum(b[:, None, 0], b[None, :, 0]))
    ih = np.maximum(0, np.minimum(b[:, None, 3], b[None, :, 3]) - np.maximum(b[:, None, 1], b[None, :, 1]))
    inter = iw * ih
    return inter / (area[:, None] + area[None, :] - inter + 1e-7)


def _greedy(over: ndarray, order: ndarray) -> list[int]:
    gone = np.zeros(len(order), bool)
    keep = []
    for i in order:
        if not gone[i]:
            keep.append(int(i))
            gone |= over[i]
    return keep


def _nms(b: ndarray, s: ndarray, iou: float) -> list[int]:
    order = np.argsort(-s, kind="stable")
    over = _iou(b) > iou
    if not over.sum() > len(b):  # only the diagonal overlaps: nothing to suppress
        return order.tolist()
    return _greedy(over, order)


def _splitmix(seed: int, n: int) -> ndarray:
    z = np.arange(1, n + 1, dtype=np.uint64) * np.uint64(0x9E3779B97F4A7C15) + np.uint64(seed)
    z = (z ^ (z >> np.uint64(30))) * np.uint64(0xBF58476D1CE4E5B9)
    z = (z ^ (z >> np.uint64(27))) * np.uint64(0x94D049BB133111EB)
    z = z ^ (z >> np.uint64(31))
    return (z >> np.uint64(11)).astype(np.float64) / 2.0 ** 53


def _stratified(seed: int, shape: tuple[int, ...]) -> ndarray:
    """Latin-hypercube uniforms along the last axis: one draw per stratum."""
    n = math.prod(shape)
    perm = np.argsort(_splitmix(seed, n).reshape(shape), axis=-1, kind="stable")
    return (perm + _splitmix(seed + 1, n).reshape(shape)) / shape[-1]


_LGF = np.array([math.lgamma(k + 1) for k in range(64)])


def _poisson(lam: float, u: ndarray) -> ndarray:
    k = np.arange(64)
    cdf = np.cumsum(np.exp(-lam + k * math.log(max(lam, 1e-12)) - _LGF))
    return np.minimum(np.searchsorted(cdf, u), 63)


def _ap_paths(t: ndarray) -> ndarray:
    """Sum of precision at each TP after keeping the top 0..n boxes, per Monte Carlo draw."""
    m, n = t.shape
    out = np.zeros((m, n + 1))
    if n:
        out[:, 1:] = np.cumsum(t * (np.cumsum(t, 1) / np.arange(1, n + 1)), 1)
    return out


class Miner:
    def __init__(self, path_hf_repo: Path) -> None:
        so = ort.SessionOptions()
        so.intra_op_num_threads = 2
        so.inter_op_num_threads = 1
        so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
        so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
        so.add_session_config_entry("session.intra_op.allow_spinning", "1")
        so.add_session_config_entry("session.inter_op.allow_spinning", "1")
        self.session = ort.InferenceSession(str(Path(path_hf_repo) / "weights.onnx"), so, providers=["CPUExecutionProvider"])
        self.input_name = self.session.get_inputs()[0].name
        self.size = int(self.session.get_inputs()[0].shape[-1])
        self.buf = np.full((1, 3, self.size, self.size), 114 / 255, dtype=np.float32)
        self.u = np.ascontiguousarray(_stratified(SEED, (3, NMAX, NS)).transpose(0, 2, 1))  # [class, draw, rank]
        self.v = _stratified(SEED + 7919, (3, NS))  # [class, draw]
        cv2.setNumThreads(1)

    def __repr__(self) -> str:
        return f"ONNX {self.size}"

    def _infer(self, image: ndarray) -> tuple[ndarray, tuple[float, float, float, int, int]]:
        h, w = image.shape[:2]
        s = self.size
        scale = min(s / h, s / w)
        nw, nh = round(w * scale), round(h * scale)
        dx, dy = (s - nw) / 2, (s - nh) / 2
        top, left = round(dy - 0.1), round(dx - 0.1)
        r = cv2.resize(image, (nw, nh)) if (nw, nh) != (w, h) else image
        inp = self.buf
        if nw != s or nh != s:
            inp.fill(114 / 255)
        k = np.float32(1 / 255)
        for c in range(3):  # BGR → RGB, HWC → CHW, /255, straight into the reused input buffer
            np.multiply(r[:, :, 2 - c], k, out=inp[0, c, top:top + nh, left:left + nw], dtype=np.float32)
        return self.session.run(None, {self.input_name: inp})[0][0].T, (scale, dx, dy, w, h)

    def _candidates(self, raw: ndarray, meta: tuple[float, float, float, int, int]):
        scale, dx, dy, w, h = meta
        conf = raw[:, 4:].max(1)
        cls = raw[:, 4:].argmax(1)
        if not len(conf) or float(conf.max()) < GATE:
            return None
        keep = conf >= FLOOR
        raw, conf, cls = raw[keep], conf[keep], cls[keep]
        if not len(raw):
            return None
        b = np.stack([raw[:, 0] - raw[:, 2] / 2, raw[:, 1] - raw[:, 3] / 2, raw[:, 0] + raw[:, 2] / 2, raw[:, 1] + raw[:, 3] / 2], 1)
        b[:, [0, 2]] = np.clip((b[:, [0, 2]] - dx) / scale, 0, w - 1)
        b[:, [1, 3]] = np.clip((b[:, [1, 3]] - dy) / scale, 0, h - 1)
        bw, bh = np.maximum(0, b[:, 2] - b[:, 0]), np.maximum(0, b[:, 3] - b[:, 1])
        ar = np.maximum(bw / np.maximum(bh, 1e-6), bh / np.maximum(bw, 1e-6))
        ok = (bw >= 3) & (bh >= 3) & (bw * bh >= 8) & (bw * bh <= 0.95 * w * h) & (ar <= 12)
        b, conf, cls = b[ok], conf[ok], cls[ok]
        if not len(conf):
            return None
        if len(conf) > PRE_TOPK:
            t = np.argpartition(-conf, PRE_TOPK)[:PRE_TOPK]
            b, conf, cls = b[t], conf[t], cls[t]
        sel = sorted(j for c in range(len(CONF)) for j in np.flatnonzero(cls == c)[_nms(b[cls == c], conf[cls == c], NMS_IOU)])
        b, conf, cls = b[sel], conf[sel], cls[sel]
        if len(conf) > MAX_DET:
            t = np.argsort(-conf)[:MAX_DET]
            b, conf, cls = b[t], conf[t], cls[t]
        if len(b) > 1:
            iw = np.maximum(0, np.minimum(b[:, None, 2], b[None, :, 2]) - np.maximum(b[:, None, 0], b[None, :, 0]))
            ih = np.maximum(0, np.minimum(b[:, None, 3], b[None, :, 3]) - np.maximum(b[:, None, 1], b[None, :, 1]))
            a_ = np.maximum(0, b[:, 2] - b[:, 0]) * np.maximum(0, b[:, 3] - b[:, 1])
            inside = iw * ih / (np.minimum(a_[:, None], a_[None, :]) + 1e-7)  # share of the smaller box inside the other
            over = (_iou(b) > DEDUP_IOU) | (inside >= 0.85)
            np.fill_diagonal(over, False)
            area = (b[:, 2] - b[:, 0]) * (b[:, 3] - b[:, 1])
            order = np.lexsort((-area, -(conf - CONF[cls])))
            kept = sorted(_greedy(over, order)) if over.any() else list(range(len(b)))
            b, conf, cls = b[kept], conf[kept], cls[kept]
        q = np.stack([np.floor(b[:, 0]), np.floor(b[:, 1]), np.ceil(b[:, 2]), np.ceil(b[:, 3])], 1)
        q[:, [0, 2]] = np.clip(q[:, [0, 2]], 0, w)
        q[:, [1, 3]] = np.clip(q[:, [1, 3]], 0, h)
        ok = (q[:, 2] > q[:, 0]) & (q[:, 3] > q[:, 1])
        return q[ok].astype(int), conf[ok], cls[ok]

    def _choose(self, q: ndarray, conf: ndarray, cls: ndarray) -> ndarray:
        ap, fp, ngt, idx = [], [], [], []
        for c in range(3):
            i = np.flatnonzero(cls == c)
            i = i[np.argsort(-conf[i], kind="stable")][:NMAX]
            idx.append(i)
            s = np.clip(conf[i].astype(np.float64), 1e-4, 1 - 1e-4)
            lg = np.log(s / (1 - s))
            la = np.log10(np.clip((q[i, 2] - q[i, 0]) * (q[i, 3] - q[i, 1]) / (1024 * 1024), 1e-6, 1))
            p = 1 / (1 + np.exp(-(CAL[c, 0] + CAL[c, 1] * lg + CAL[c, 2] * la + CAL[c, 3] * lg * la + SHIFT)))
            t = self.u[c][:, :len(i)] < p[None, :]  # [draw, rank]: is the box a TP in this draw
            ap.append(_ap_paths(t))
            fp.append(np.arange(len(i) + 1)[None, :] - np.concatenate([np.zeros((NS, 1)), np.cumsum(t, 1)], 1))
            ngt.append(t.sum(1) + _poisson(max(MISS[c, 0] + MISS[c, 1] * float(p.sum()), 0.0), self.v[c]))
        ngt = np.stack(ngt, 1)
        present = ngt > 0
        n_present = np.maximum(present.sum(1), 1)
        empty = present.sum(1) == 0
        part = [np.where(present[:, c, None], ap[c] / np.maximum(ngt[:, c, None], 1), 0.0) for c in range(3)]
        k = [0, 0, 0]
        a_tot = sum(part[c][:, 0] for c in range(3))
        f_tot = sum(fp[c][:, 0] for c in range(3))

        def value(c: int) -> ndarray:  # expected max(score - THETA, .01) for every k of class c, others fixed
            a = (a_tot - part[c][:, k[c]])[:, None] + part[c]
            f = (f_tot - fp[c][:, k[c]])[:, None] + fp[c]
            n = sum(k) - k[c] + np.arange(len(idx[c]) + 1)
            pos = np.maximum(0.6 * a / n_present[:, None] + 0.4 * np.maximum(0.0, 1 - f / 10) - THETA, 0.01)
            neg = np.maximum(np.maximum(0.0, 1 - n / 10) - THETA, 0.01)
            return np.where(empty[:, None], neg[None, :], pos).mean(0)

        best = value(0)[0]
        for _ in range(3):  # coordinate ascent over the per-class counts
            moved = False
            for c in range(3):
                v = value(c)
                j = int(np.argmax(v))
                if v[j] > best + 1e-9:
                    a_tot += part[c][:, j] - part[c][:, k[c]]
                    f_tot += fp[c][:, j] - fp[c][:, k[c]]
                    best, k[c], moved = float(v[j]), j, True
            if not moved:
                break
        return np.concatenate([idx[c][:k[c]] for c in range(3)]).astype(int)

    def _select(self, raw: ndarray, meta: tuple[float, float, float, int, int]) -> list[BoundingBox]:
        got = self._candidates(raw, meta)
        if got is None:
            return []
        q, conf, cls = got
        sel = self._choose(q, conf, cls)
        return [BoundingBox(x1=int(q[i, 0]), y1=int(q[i, 1]), x2=int(q[i, 2]), y2=int(q[i, 3]), cls_id=int(cls[i]),
                            conf=float(min(1.0, max(0.0, conf[i])))) for i in sorted(sel.tolist(), key=lambda i: -conf[i])]

    def _detect(self, image: ndarray) -> list[BoundingBox]:
        try:
            return self._select(*self._infer(image))
        except Exception as e:  # one bad frame must not fail the whole batch
            print(f"frame failed: {e}")
            return []

    def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]:
        return [TVFrameResult(frame_id=offset + i, boxes=self._detect(im), keypoints=[(0, 0)] * n_keypoints)
                for i, im in enumerate(batch_images)]