coolroman commited on
Commit
8767c8b
·
verified ·
1 Parent(s): 1f86cfe

scorevision: push artifact

Browse files
Files changed (1) hide show
  1. miner.py +637 -434
miner.py CHANGED
@@ -1,434 +1,637 @@
1
- """Plate-detection miner v2.0 "hermestech + tile s<3".
2
-
3
- Base weights: hermestech00/numberplate0 (YOLO26s retrained, fp16, ~19 MB).
4
-
5
- Inference pipeline:
6
- 1) Full-image primary pass with alfred001 tuning
7
- (conf=0.22, iou=0.41, sigma=0.685, soft-NMS + hflip TTA).
8
- 2) If the primary returned fewer than 3 boxes, run a 2x2
9
- overlapping tile pass (tile_conf=0.40) with novelty-merge.
10
- """
11
- from pathlib import Path
12
- import math
13
-
14
- import cv2
15
- import numpy as np
16
- import onnxruntime as ort
17
- from numpy import ndarray
18
- from pydantic import BaseModel
19
-
20
-
21
- class BoundingBox(BaseModel):
22
- x1: int
23
- y1: int
24
- x2: int
25
- y2: int
26
- cls_id: int
27
- conf: float
28
-
29
-
30
- class TVFrameResult(BaseModel):
31
- frame_id: int
32
- boxes: list[BoundingBox]
33
- keypoints: list[tuple[int, int]]
34
-
35
-
36
- SIZE = 1280
37
-
38
-
39
- class Miner:
40
- def __init__(self, path_hf_repo: Path) -> None:
41
- model_path = path_hf_repo / "weights.onnx"
42
- cn_path = model_path.with_name("class_names.txt")
43
- if cn_path.is_file():
44
- lines = cn_path.read_text(encoding="utf-8").splitlines()
45
- self.class_names = [
46
- ln.strip()
47
- for ln in lines
48
- if ln.strip() and not ln.strip().startswith("#")
49
- ]
50
- else:
51
- self.class_names = ["numberplate"]
52
- print("ORT version:", ort.__version__)
53
-
54
- try:
55
- ort.preload_dlls()
56
- print("onnxruntime.preload_dlls() success")
57
- except Exception as e:
58
- print(f"preload_dlls failed: {e}")
59
-
60
- print("ORT available providers BEFORE session:", ort.get_available_providers())
61
-
62
- try:
63
- import torch
64
- if torch.cuda.is_available():
65
- print(f"GPU: {torch.cuda.get_device_name(0)}")
66
- print(f"GPU memory: {torch.cuda.get_device_properties(0).total_mem / 1e9:.1f} GB")
67
- else:
68
- print("GPU: CUDA not available via torch")
69
- except Exception as e:
70
- print(f"GPU detection failed: {e}")
71
-
72
- sess_options = ort.SessionOptions()
73
- sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
74
-
75
- try:
76
- self.session = ort.InferenceSession(
77
- str(model_path),
78
- sess_options=sess_options,
79
- providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
80
- )
81
- print("Created ORT session with preferred CUDA provider list")
82
- except Exception as e:
83
- print(f"CUDA session creation failed, falling back to CPU: {e}")
84
- self.session = ort.InferenceSession(
85
- str(model_path),
86
- sess_options=sess_options,
87
- providers=["CPUExecutionProvider"],
88
- )
89
-
90
- print("ORT session providers:", self.session.get_providers())
91
-
92
- for inp in self.session.get_inputs():
93
- print("INPUT:", inp.name, inp.shape, inp.type)
94
- for out in self.session.get_outputs():
95
- print("OUTPUT:", out.name, out.shape, out.type)
96
-
97
- self.input_name = self.session.get_inputs()[0].name
98
- self.output_names = [o.name for o in self.session.get_outputs()]
99
- self.input_shape = self.session.get_inputs()[0].shape
100
-
101
- self.input_height = self._safe_dim(self.input_shape[2], default=SIZE)
102
- self.input_width = self._safe_dim(self.input_shape[3], default=SIZE)
103
-
104
- # Primary pass: alfred001 tuning (optimized for hermestech weights)
105
- self.conf_thres = 0.22
106
- self.iou_thres = 0.41
107
- self.sigma = 0.685
108
- self.max_det = 300
109
-
110
- # Conditional tile-pass (trimmed for latency: no hflip, tighter sparse)
111
- self.sparse_threshold = 3 # fire tiles only if primary returns < this
112
- self.tile_conf = 0.40
113
- self.tile_overlap = 0.20
114
- self.novelty_iou = 0.10
115
- self.final_max_det = 22
116
- self.tile_use_hflip = False # skip hflip tile pass to save ~4 forwards
117
-
118
- self.use_tta = True
119
-
120
- print(f"ONNX model loaded from: {model_path}")
121
- print(f"ONNX providers: {self.session.get_providers()}")
122
- print(f"ONNX input: name={self.input_name}, shape={self.input_shape}")
123
-
124
- def __repr__(self) -> str:
125
- return (
126
- f"ONNXRuntime(session={type(self.session).__name__}, "
127
- f"providers={self.session.get_providers()})"
128
- )
129
-
130
- @staticmethod
131
- def _safe_dim(value, default: int) -> int:
132
- return value if isinstance(value, int) and value > 0 else default
133
-
134
- # ---------- image preprocessing ----------
135
- def _letterbox(
136
- self,
137
- image: ndarray,
138
- new_shape: tuple[int, int],
139
- color=(114, 114, 114),
140
- ) -> tuple[ndarray, float, tuple[float, float]]:
141
- h, w = image.shape[:2]
142
- new_w, new_h = new_shape
143
- ratio = min(new_w / w, new_h / h)
144
- resized_w = int(round(w * ratio))
145
- resized_h = int(round(h * ratio))
146
- if (resized_w, resized_h) != (w, h):
147
- interp = cv2.INTER_CUBIC if ratio > 1.0 else cv2.INTER_LINEAR
148
- image = cv2.resize(image, (resized_w, resized_h), interpolation=interp)
149
- dw = (new_w - resized_w) / 2.0
150
- dh = (new_h - resized_h) / 2.0
151
- left = int(round(dw - 0.1))
152
- right = int(round(dw + 0.1))
153
- top = int(round(dh - 0.1))
154
- bottom = int(round(dh + 0.1))
155
- padded = cv2.copyMakeBorder(
156
- image, top, bottom, left, right,
157
- borderType=cv2.BORDER_CONSTANT, value=color,
158
- )
159
- return padded, ratio, (dw, dh)
160
-
161
- def _preprocess(self, image: ndarray):
162
- img, ratio, pad = self._letterbox(image, (self.input_width, self.input_height))
163
- img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0
164
- img = np.transpose(img, (2, 0, 1))[None, ...]
165
- return np.ascontiguousarray(img, dtype=np.float32), ratio, pad
166
-
167
- @staticmethod
168
- def _clip_boxes(boxes: np.ndarray, image_size: tuple[int, int]) -> np.ndarray:
169
- w, h = image_size
170
- boxes[:, 0] = np.clip(boxes[:, 0], 0, w - 1)
171
- boxes[:, 1] = np.clip(boxes[:, 1], 0, h - 1)
172
- boxes[:, 2] = np.clip(boxes[:, 2], 0, w - 1)
173
- boxes[:, 3] = np.clip(boxes[:, 3], 0, h - 1)
174
- return boxes
175
-
176
- # ---------- NMS primitives ----------
177
- @staticmethod
178
- def _hard_nms(boxes: np.ndarray, scores: np.ndarray, iou_thresh: float) -> np.ndarray:
179
- N = len(boxes)
180
- if N == 0:
181
- return np.array([], dtype=np.intp)
182
- boxes = np.asarray(boxes, dtype=np.float32)
183
- scores = np.asarray(scores, dtype=np.float32)
184
- order = np.argsort(-scores)
185
- keep: list[int] = []
186
- while len(order):
187
- i = int(order[0])
188
- keep.append(i)
189
- if len(order) == 1:
190
- break
191
- rest = order[1:]
192
- xx1 = np.maximum(boxes[i, 0], boxes[rest, 0])
193
- yy1 = np.maximum(boxes[i, 1], boxes[rest, 1])
194
- xx2 = np.minimum(boxes[i, 2], boxes[rest, 2])
195
- yy2 = np.minimum(boxes[i, 3], boxes[rest, 3])
196
- inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
197
- area_i = (boxes[i, 2] - boxes[i, 0]) * (boxes[i, 3] - boxes[i, 1])
198
- area_r = (boxes[rest, 2] - boxes[rest, 0]) * (boxes[rest, 3] - boxes[rest, 1])
199
- iou = inter / (area_i + area_r - inter + 1e-7)
200
- order = rest[iou <= iou_thresh]
201
- return np.array(keep, dtype=np.intp)
202
-
203
- def _soft_nms(
204
- self,
205
- boxes: np.ndarray,
206
- scores: np.ndarray,
207
- sigma: float,
208
- score_thresh: float = 0.01,
209
- ) -> tuple[np.ndarray, np.ndarray]:
210
- N = len(boxes)
211
- if N == 0:
212
- return np.array([], dtype=np.intp), np.array([], dtype=np.float32)
213
- boxes = boxes.astype(np.float32, copy=True)
214
- scores = scores.astype(np.float32, copy=True)
215
- order = np.arange(N)
216
- for i in range(N):
217
- max_pos = i + int(np.argmax(scores[i:]))
218
- boxes[[i, max_pos]] = boxes[[max_pos, i]]
219
- scores[[i, max_pos]] = scores[[max_pos, i]]
220
- order[[i, max_pos]] = order[[max_pos, i]]
221
- if i + 1 >= N:
222
- break
223
- xx1 = np.maximum(boxes[i, 0], boxes[i + 1:, 0])
224
- yy1 = np.maximum(boxes[i, 1], boxes[i + 1:, 1])
225
- xx2 = np.minimum(boxes[i, 2], boxes[i + 1:, 2])
226
- yy2 = np.minimum(boxes[i, 3], boxes[i + 1:, 3])
227
- inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
228
- area_i = float(
229
- (boxes[i, 2] - boxes[i, 0]) * (boxes[i, 3] - boxes[i, 1])
230
- )
231
- areas_j = (
232
- np.maximum(0.0, boxes[i + 1:, 2] - boxes[i + 1:, 0])
233
- * np.maximum(0.0, boxes[i + 1:, 3] - boxes[i + 1:, 1])
234
- )
235
- iou = inter / (area_i + areas_j - inter + 1e-7)
236
- scores[i + 1:] *= np.exp(-(iou ** 2) / sigma)
237
- mask = scores > score_thresh
238
- return order[mask], scores[mask]
239
-
240
- @staticmethod
241
- def _box_iou_one_to_many(box: np.ndarray, boxes: np.ndarray) -> np.ndarray:
242
- if len(boxes) == 0:
243
- return np.zeros(0, dtype=np.float32)
244
- xx1 = np.maximum(box[0], boxes[:, 0])
245
- yy1 = np.maximum(box[1], boxes[:, 1])
246
- xx2 = np.minimum(box[2], boxes[:, 2])
247
- yy2 = np.minimum(box[3], boxes[:, 3])
248
- inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
249
- area_a = max(0.0, (box[2] - box[0]) * (box[3] - box[1]))
250
- area_b = np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) * np.maximum(0.0, boxes[:, 3] - boxes[:, 1])
251
- return inter / (area_a + area_b - inter + 1e-7)
252
-
253
- # ---------- raw-dets helper ----------
254
- def _raw_dets(self, image: ndarray, conf: float) -> np.ndarray:
255
- """Run a single forward pass and return [N, 5] dets in ORIGINAL image coords."""
256
- x, ratio, (dw, dh) = self._preprocess(image)
257
- out = self.session.run(self.output_names, {self.input_name: x})[0]
258
- if out.ndim == 3:
259
- out = out[0]
260
- if out.shape[1] < 5:
261
- return np.zeros((0, 5), dtype=np.float32)
262
- boxes = out[:, :4].astype(np.float32)
263
- scores = out[:, 4].astype(np.float32)
264
- keep = scores >= conf
265
- boxes, scores = boxes[keep], scores[keep]
266
- if len(boxes) == 0:
267
- return np.zeros((0, 5), dtype=np.float32)
268
- boxes[:, [0, 2]] -= dw
269
- boxes[:, [1, 3]] -= dh
270
- boxes /= ratio
271
- oh, ow = image.shape[:2]
272
- boxes = self._clip_boxes(boxes, (ow, oh))
273
- return np.concatenate([boxes, scores[:, None]], axis=1)
274
-
275
- # ---------- primary pass: soft-NMS + hflip TTA ----------
276
- def _primary(self, image: ndarray) -> np.ndarray:
277
- d1 = self._raw_dets(image, self.conf_thres)
278
- flipped = cv2.flip(image, 1)
279
- d2 = self._raw_dets(flipped, self.conf_thres)
280
- if len(d2):
281
- w = image.shape[1]
282
- x1 = w - d2[:, 2]
283
- x2 = w - d2[:, 0]
284
- d2 = np.stack([x1, d2[:, 1], x2, d2[:, 3], d2[:, 4]], axis=1)
285
- all_d = np.concatenate([d1, d2], axis=0) if len(d2) else d1
286
- if len(all_d) == 0:
287
- return np.zeros((0, 5), dtype=np.float32)
288
- # soft-NMS, then hard-NMS
289
- keep_idx, scores = self._soft_nms(all_d[:, :4].copy(), all_d[:, 4].copy(), sigma=self.sigma)
290
- if len(keep_idx) == 0:
291
- return np.zeros((0, 5), dtype=np.float32)
292
- merged = np.concatenate([all_d[keep_idx, :4], scores[:, None]], axis=1)
293
- keep = self._hard_nms(merged[:, :4], merged[:, 4], self.iou_thres)
294
- merged = merged[keep]
295
- if len(merged) > self.max_det:
296
- merged = merged[np.argsort(-merged[:, 4])[: self.max_det]]
297
- return merged
298
-
299
- # ---------- conditional tile pass ----------
300
- def _tile_augment(self, image: ndarray, primary: np.ndarray) -> np.ndarray:
301
- """Run 2x2 overlapping tiles + hflip, novelty-merge into primary."""
302
- oh, ow = image.shape[:2]
303
- tw, th = ow // 2, oh // 2
304
- ox, oy = int(tw * self.tile_overlap), int(th * self.tile_overlap)
305
- tiles = [
306
- (0, 0, min(ow, tw + ox), min(oh, th + oy)),
307
- (max(0, tw - ox), 0, ow, min(oh, th + oy)),
308
- (0, max(0, th - oy), min(ow, tw + ox), oh),
309
- (max(0, tw - ox), max(0, th - oy), ow, oh),
310
- ]
311
- collected: list[np.ndarray] = []
312
- for x1, y1, x2, y2 in tiles:
313
- crop = image[y1:y2, x1:x2]
314
- if crop.size == 0:
315
- continue
316
- d = self._raw_dets(crop, self.tile_conf)
317
- if len(d):
318
- d[:, 0] += x1
319
- d[:, 1] += y1
320
- d[:, 2] += x1
321
- d[:, 3] += y1
322
- collected.append(d)
323
-
324
- # hflip tile pass (skipped when tile_use_hflip=False — saves 4 ONNX forwards)
325
- if self.tile_use_hflip:
326
- flipped = cv2.flip(image, 1)
327
- for x1, y1, x2, y2 in tiles:
328
- fx1 = ow - x2
329
- fx2 = ow - x1
330
- if fx2 <= fx1:
331
- continue
332
- crop = flipped[y1:y2, fx1:fx2]
333
- if crop.size == 0:
334
- continue
335
- d = self._raw_dets(crop, self.tile_conf)
336
- if len(d):
337
- d_un = d.copy()
338
- d_un[:, 0] = (ow - (d[:, 2] + fx1))
339
- d_un[:, 2] = (ow - (d[:, 0] + fx1))
340
- d_un[:, 1] = d[:, 1] + y1
341
- d_un[:, 3] = d[:, 3] + y1
342
- collected.append(d_un)
343
-
344
- if not collected:
345
- return primary
346
-
347
- tile_dets = np.concatenate(collected, axis=0)
348
- keep = self._hard_nms(tile_dets[:, :4], tile_dets[:, 4], 0.5)
349
- tile_dets = tile_dets[keep]
350
-
351
- # Novelty: drop tile boxes that overlap any primary box at IoU >= novelty_iou
352
- if len(primary) > 0 and len(tile_dets) > 0:
353
- mask = np.ones(len(tile_dets), dtype=bool)
354
- for i in range(len(tile_dets)):
355
- ious = self._box_iou_one_to_many(tile_dets[i, :4], primary[:, :4])
356
- if len(ious) and np.max(ious) >= self.novelty_iou:
357
- mask[i] = False
358
- tile_dets = tile_dets[mask]
359
-
360
- if len(tile_dets) == 0:
361
- return primary
362
-
363
- # Sanity filter: min/max size, aspect ratio
364
- w = tile_dets[:, 2] - tile_dets[:, 0]
365
- h = tile_dets[:, 3] - tile_dets[:, 1]
366
- area = w * h
367
- ar = np.maximum(w / np.maximum(h, 1e-6), h / np.maximum(w, 1e-6))
368
- img_area = float(ow * oh)
369
- ok = (w >= 6) & (h >= 6) & (area >= 36) & (area <= 0.5 * img_area) & (ar <= 10.0)
370
- tile_dets = tile_dets[ok]
371
- if len(tile_dets) == 0:
372
- return primary
373
-
374
- merged = np.concatenate([primary, tile_dets], axis=0)
375
- keep = self._hard_nms(merged[:, :4], merged[:, 4], self.iou_thres)
376
- merged = merged[keep]
377
- if len(merged) > self.final_max_det:
378
- merged = merged[np.argsort(-merged[:, 4])[: self.final_max_det]]
379
- return merged
380
-
381
- # ---------- single-image predict ----------
382
- def _predict_single(self, image: ndarray) -> list[BoundingBox]:
383
- if image is None or not isinstance(image, np.ndarray) or image.ndim != 3:
384
- return []
385
- if image.shape[0] <= 0 or image.shape[1] <= 0 or image.shape[2] != 3:
386
- return []
387
- if image.dtype != np.uint8:
388
- image = image.astype(np.uint8)
389
-
390
- primary = self._primary(image)
391
- if len(primary) < self.sparse_threshold:
392
- dets = self._tile_augment(image, primary)
393
- else:
394
- dets = primary
395
-
396
- results: list[BoundingBox] = []
397
- for row in dets:
398
- x1, y1, x2, y2, conf = row.tolist()
399
- if x2 <= x1 or y2 <= y1:
400
- continue
401
- results.append(
402
- BoundingBox(
403
- x1=int(math.floor(x1)),
404
- y1=int(math.floor(y1)),
405
- x2=int(math.ceil(x2)),
406
- y2=int(math.ceil(y2)),
407
- cls_id=0,
408
- conf=float(conf),
409
- )
410
- )
411
- return results
412
-
413
- # ---------- chute entrypoint ----------
414
- def predict_batch(
415
- self,
416
- batch_images: list[ndarray],
417
- offset: int,
418
- n_keypoints: int,
419
- ) -> list[TVFrameResult]:
420
- results: list[TVFrameResult] = []
421
- for frame_number_in_batch, image in enumerate(batch_images):
422
- try:
423
- boxes = self._predict_single(image)
424
- except Exception as e:
425
- print(f"Inference failed for frame {offset + frame_number_in_batch}: {e}")
426
- boxes = []
427
- results.append(
428
- TVFrameResult(
429
- frame_id=offset + frame_number_in_batch,
430
- boxes=boxes,
431
- keypoints=[(0, 0) for _ in range(max(0, int(n_keypoints)))],
432
- )
433
- )
434
- return results
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Person detection miner for TurboVision element manak0/Detect-Person.
2
+ # Seeded from the current champion alfred8995/york004 (rev 79eec28…) as a
3
+ # well-proven scoring-aware pipeline. Differentiation is expected to come from:
4
+ # 1. A stronger base detector (YOLO26-s / YOLO11-m vs champion's YOLOv11-nano)
5
+ # 2. Better training data
6
+ # 3. Post-training threshold sweep on our held-out synthetic set
7
+ # Inference contract (inputs/outputs + weights.onnx filename) is locked to the
8
+ # turbovision chute template do not rename without updating chute_config.yml.
9
+ from pathlib import Path
10
+ import math
11
+
12
+ import cv2
13
+ import numpy as np
14
+ import onnxruntime as ort
15
+ from numpy import ndarray
16
+ from pydantic import BaseModel
17
+
18
+
19
+ class BoundingBox(BaseModel):
20
+ x1: int
21
+ y1: int
22
+ x2: int
23
+ y2: int
24
+ cls_id: int
25
+ conf: float
26
+
27
+
28
+ class TVFrameResult(BaseModel):
29
+ frame_id: int
30
+ boxes: list[BoundingBox]
31
+ keypoints: list[tuple[int, int]]
32
+
33
+
34
+ class Miner:
35
+ def __init__(self, path_hf_repo: Path) -> None:
36
+ model_path = path_hf_repo / "weights.onnx"
37
+ self.class_names = ["person"]
38
+ print("ORT version:", ort.__version__)
39
+
40
+ try:
41
+ ort.preload_dlls()
42
+ print("✅ onnxruntime.preload_dlls() success")
43
+ except Exception as e:
44
+ print(f"⚠️ preload_dlls failed: {e}")
45
+
46
+ print("ORT available providers BEFORE session:", ort.get_available_providers())
47
+
48
+ sess_options = ort.SessionOptions()
49
+ sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
50
+
51
+ try:
52
+ self.session = ort.InferenceSession(
53
+ str(model_path),
54
+ sess_options=sess_options,
55
+ providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
56
+ )
57
+ print("✅ Created ORT session with preferred CUDA provider list")
58
+ except Exception as e:
59
+ print(f"⚠️ CUDA session creation failed, falling back to CPU: {e}")
60
+ self.session = ort.InferenceSession(
61
+ str(model_path),
62
+ sess_options=sess_options,
63
+ providers=["CPUExecutionProvider"],
64
+ )
65
+
66
+ print("ORT session providers:", self.session.get_providers())
67
+
68
+ for inp in self.session.get_inputs():
69
+ print("INPUT:", inp.name, inp.shape, inp.type)
70
+
71
+ for out in self.session.get_outputs():
72
+ print("OUTPUT:", out.name, out.shape, out.type)
73
+
74
+ self.input_name = self.session.get_inputs()[0].name
75
+ self.output_names = [output.name for output in self.session.get_outputs()]
76
+ self.input_shape = self.session.get_inputs()[0].shape
77
+
78
+ self.input_height = self._safe_dim(self.input_shape[2], default=1280)
79
+ self.input_width = self._safe_dim(self.input_shape[3], default=1280)
80
+
81
+ # --- Scoring-aware adaptive confidence ---
82
+ # total_score = mAP50 * 0.65 + FP_score * 0.35
83
+ # FP_score = max(0, 1 - n_FP / n_images), typically n_images ≈ 10
84
+ #
85
+ # v12s thresholds: swept on SAM3 GT bench (276 imgs). v12s = yolo26-s
86
+ # student distilled from v12-m teacher (yolo26-m trained on hermestech-
87
+ # consensus TV labels + 14k aux CCTV/crowd images relabeled by v12-m).
88
+ # Swept composite 0.8750 vs hermestech 0.8371 (+0.038 lead).
89
+ self.conf_thres = 0.2149 # Base threshold for candidate generation
90
+ self.iou_thres = 0.4704 # NMS threshold
91
+ self.max_det = 150
92
+
93
+ # TTA consensus thresholds
94
+ self.conf_high = 0.7443 # Boxes above this survive without TTA confirmation
95
+ self.tta_match_iou = 0.4447 # TTA cross-view match IoU
96
+
97
+ # Adaptive conf curve: lerp between low/high based on raw detection count
98
+ self.conf_adapt_low = 0.1467 # Few objects: favor recall
99
+ self.conf_adapt_high = 0.6997 # Many objects: favor precision
100
+ self.count_low = 9 # Raw count below this → use conf_adapt_low
101
+ self.count_high = 46 # Raw count above this → use conf_adapt_high
102
+
103
+ self.use_tta = True
104
+
105
+ # Box sanity filters
106
+ self.min_box_area = 14 * 14
107
+ self.min_w = 8
108
+ self.min_h = 8
109
+ self.max_aspect_ratio = 6.5
110
+ self.max_box_area_ratio = 0.8
111
+
112
+ print(f"✅ ONNX model loaded from: {model_path}")
113
+ print(f"✅ ONNX providers: {self.session.get_providers()}")
114
+ print(f"✅ ONNX input: name={self.input_name}, shape={self.input_shape}")
115
+
116
+ def __repr__(self) -> str:
117
+ return (
118
+ f"ONNXRuntime(session={type(self.session).__name__}, "
119
+ f"providers={self.session.get_providers()})"
120
+ )
121
+
122
+ @staticmethod
123
+ def _safe_dim(value, default: int) -> int:
124
+ return value if isinstance(value, int) and value > 0 else default
125
+
126
+ def _letterbox(
127
+ self,
128
+ image: ndarray,
129
+ new_shape: tuple[int, int],
130
+ color=(114, 114, 114),
131
+ ) -> tuple[ndarray, float, tuple[float, float]]:
132
+ h, w = image.shape[:2]
133
+ new_w, new_h = new_shape
134
+
135
+ ratio = min(new_w / w, new_h / h)
136
+ resized_w = int(round(w * ratio))
137
+ resized_h = int(round(h * ratio))
138
+
139
+ if (resized_w, resized_h) != (w, h):
140
+ interp = cv2.INTER_CUBIC if ratio > 1.0 else cv2.INTER_LINEAR
141
+ image = cv2.resize(image, (resized_w, resized_h), interpolation=interp)
142
+
143
+ dw = new_w - resized_w
144
+ dh = new_h - resized_h
145
+ dw /= 2.0
146
+ dh /= 2.0
147
+
148
+ left = int(round(dw - 0.1))
149
+ right = int(round(dw + 0.1))
150
+ top = int(round(dh - 0.1))
151
+ bottom = int(round(dh + 0.1))
152
+
153
+ padded = cv2.copyMakeBorder(
154
+ image,
155
+ top,
156
+ bottom,
157
+ left,
158
+ right,
159
+ borderType=cv2.BORDER_CONSTANT,
160
+ value=color,
161
+ )
162
+ return padded, ratio, (dw, dh)
163
+
164
+ def _preprocess(
165
+ self, image: ndarray
166
+ ) -> tuple[np.ndarray, float, tuple[float, float], tuple[int, int]]:
167
+ orig_h, orig_w = image.shape[:2]
168
+
169
+ img, ratio, pad = self._letterbox(
170
+ image, (self.input_width, self.input_height)
171
+ )
172
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
173
+ img = img.astype(np.float32) / 255.0
174
+ img = np.transpose(img, (2, 0, 1))[None, ...]
175
+ img = np.ascontiguousarray(img, dtype=np.float32)
176
+
177
+ return img, ratio, pad, (orig_w, orig_h)
178
+
179
+ @staticmethod
180
+ def _clip_boxes(boxes: np.ndarray, image_size: tuple[int, int]) -> np.ndarray:
181
+ w, h = image_size
182
+ boxes[:, 0] = np.clip(boxes[:, 0], 0, w - 1)
183
+ boxes[:, 1] = np.clip(boxes[:, 1], 0, h - 1)
184
+ boxes[:, 2] = np.clip(boxes[:, 2], 0, w - 1)
185
+ boxes[:, 3] = np.clip(boxes[:, 3], 0, h - 1)
186
+ return boxes
187
+
188
+ @staticmethod
189
+ def _xywh_to_xyxy(boxes: np.ndarray) -> np.ndarray:
190
+ out = np.empty_like(boxes)
191
+ out[:, 0] = boxes[:, 0] - boxes[:, 2] / 2.0
192
+ out[:, 1] = boxes[:, 1] - boxes[:, 3] / 2.0
193
+ out[:, 2] = boxes[:, 0] + boxes[:, 2] / 2.0
194
+ out[:, 3] = boxes[:, 1] + boxes[:, 3] / 2.0
195
+ return out
196
+
197
+ @staticmethod
198
+ def _hard_nms(
199
+ boxes: np.ndarray,
200
+ scores: np.ndarray,
201
+ iou_thresh: float,
202
+ ) -> np.ndarray:
203
+ if len(boxes) == 0:
204
+ return np.array([], dtype=np.intp)
205
+
206
+ boxes = np.asarray(boxes, dtype=np.float32)
207
+ scores = np.asarray(scores, dtype=np.float32)
208
+ order = np.argsort(scores)[::-1]
209
+ keep = []
210
+
211
+ while len(order) > 0:
212
+ i = order[0]
213
+ keep.append(i)
214
+ if len(order) == 1:
215
+ break
216
+
217
+ rest = order[1:]
218
+
219
+ xx1 = np.maximum(boxes[i, 0], boxes[rest, 0])
220
+ yy1 = np.maximum(boxes[i, 1], boxes[rest, 1])
221
+ xx2 = np.minimum(boxes[i, 2], boxes[rest, 2])
222
+ yy2 = np.minimum(boxes[i, 3], boxes[rest, 3])
223
+
224
+ inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
225
+
226
+ area_i = np.maximum(0.0, (boxes[i, 2] - boxes[i, 0])) * np.maximum(0.0, (boxes[i, 3] - boxes[i, 1]))
227
+ area_r = np.maximum(0.0, (boxes[rest, 2] - boxes[rest, 0])) * np.maximum(0.0, (boxes[rest, 3] - boxes[rest, 1]))
228
+
229
+ iou = inter / (area_i + area_r - inter + 1e-7)
230
+ order = rest[iou <= iou_thresh]
231
+
232
+ return np.array(keep, dtype=np.intp)
233
+
234
+ @staticmethod
235
+ def _box_iou_one_to_many(box: np.ndarray, boxes: np.ndarray) -> np.ndarray:
236
+ xx1 = np.maximum(box[0], boxes[:, 0])
237
+ yy1 = np.maximum(box[1], boxes[:, 1])
238
+ xx2 = np.minimum(box[2], boxes[:, 2])
239
+ yy2 = np.minimum(box[3], boxes[:, 3])
240
+
241
+ inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
242
+
243
+ area_a = max(0.0, (box[2] - box[0]) * (box[3] - box[1]))
244
+ area_b = np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) * np.maximum(0.0, boxes[:, 3] - boxes[:, 1])
245
+
246
+ return inter / (area_a + area_b - inter + 1e-7)
247
+
248
+ def _filter_sane_boxes(
249
+ self,
250
+ boxes: np.ndarray,
251
+ scores: np.ndarray,
252
+ cls_ids: np.ndarray,
253
+ orig_size: tuple[int, int],
254
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
255
+ if len(boxes) == 0:
256
+ return boxes, scores, cls_ids
257
+
258
+ orig_w, orig_h = orig_size
259
+ image_area = float(orig_w * orig_h)
260
+
261
+ keep = []
262
+ for i, box in enumerate(boxes):
263
+ x1, y1, x2, y2 = box.tolist()
264
+ bw = x2 - x1
265
+ bh = y2 - y1
266
+
267
+ if bw <= 0 or bh <= 0:
268
+ continue
269
+ if bw < self.min_w or bh < self.min_h:
270
+ continue
271
+
272
+ area = bw * bh
273
+ if area < self.min_box_area:
274
+ continue
275
+ if area > self.max_box_area_ratio * image_area:
276
+ continue
277
+
278
+ ar = max(bw / max(bh, 1e-6), bh / max(bw, 1e-6))
279
+ if ar > self.max_aspect_ratio:
280
+ continue
281
+
282
+ keep.append(i)
283
+
284
+ if not keep:
285
+ return (
286
+ np.empty((0, 4), dtype=np.float32),
287
+ np.empty((0,), dtype=np.float32),
288
+ np.empty((0,), dtype=np.int32),
289
+ )
290
+
291
+ keep = np.array(keep, dtype=np.intp)
292
+ return boxes[keep], scores[keep], cls_ids[keep]
293
+
294
+ def _decode_final_dets(
295
+ self,
296
+ preds: np.ndarray,
297
+ ratio: float,
298
+ pad: tuple[float, float],
299
+ orig_size: tuple[int, int],
300
+ ) -> list[BoundingBox]:
301
+ if preds.ndim == 3 and preds.shape[0] == 1:
302
+ preds = preds[0]
303
+
304
+ if preds.ndim != 2 or preds.shape[1] < 6:
305
+ raise ValueError(f"Unexpected ONNX final-det output shape: {preds.shape}")
306
+
307
+ boxes = preds[:, :4].astype(np.float32)
308
+ scores = preds[:, 4].astype(np.float32)
309
+ cls_ids = preds[:, 5].astype(np.int32)
310
+
311
+ # person only
312
+ keep = cls_ids == 0
313
+ boxes = boxes[keep]
314
+ scores = scores[keep]
315
+ cls_ids = cls_ids[keep]
316
+
317
+ # candidate threshold
318
+ keep = scores >= self.conf_thres
319
+ boxes = boxes[keep]
320
+ scores = scores[keep]
321
+ cls_ids = cls_ids[keep]
322
+
323
+ if len(boxes) == 0:
324
+ return []
325
+
326
+ pad_w, pad_h = pad
327
+ orig_w, orig_h = orig_size
328
+
329
+ boxes[:, [0, 2]] -= pad_w
330
+ boxes[:, [1, 3]] -= pad_h
331
+ boxes /= ratio
332
+ boxes = self._clip_boxes(boxes, (orig_w, orig_h))
333
+
334
+ boxes, scores, cls_ids = self._filter_sane_boxes(boxes, scores, cls_ids, orig_size)
335
+ if len(boxes) == 0:
336
+ return []
337
+
338
+ keep_idx = self._hard_nms(boxes, scores, self.iou_thres)
339
+ keep_idx = keep_idx[: self.max_det]
340
+
341
+ boxes = boxes[keep_idx]
342
+ scores = scores[keep_idx]
343
+ cls_ids = cls_ids[keep_idx]
344
+
345
+ return [
346
+ BoundingBox(
347
+ x1=int(math.floor(box[0])),
348
+ y1=int(math.floor(box[1])),
349
+ x2=int(math.ceil(box[2])),
350
+ y2=int(math.ceil(box[3])),
351
+ cls_id=int(cls_id),
352
+ conf=float(conf),
353
+ )
354
+ for box, conf, cls_id in zip(boxes, scores, cls_ids)
355
+ if box[2] > box[0] and box[3] > box[1]
356
+ ]
357
+
358
+ def _decode_raw_yolo(
359
+ self,
360
+ preds: np.ndarray,
361
+ ratio: float,
362
+ pad: tuple[float, float],
363
+ orig_size: tuple[int, int],
364
+ ) -> list[BoundingBox]:
365
+ if preds.ndim != 3:
366
+ raise ValueError(f"Unexpected raw ONNX output shape: {preds.shape}")
367
+ if preds.shape[0] != 1:
368
+ raise ValueError(f"Unexpected batch dimension in raw output: {preds.shape}")
369
+
370
+ preds = preds[0]
371
+
372
+ # Normalize to [N, C]
373
+ if preds.shape[0] <= 16 and preds.shape[1] > preds.shape[0]:
374
+ preds = preds.T
375
+
376
+ if preds.ndim != 2 or preds.shape[1] < 5:
377
+ raise ValueError(f"Unexpected normalized raw output shape: {preds.shape}")
378
+
379
+ boxes_xywh = preds[:, :4].astype(np.float32)
380
+ tail = preds[:, 4:].astype(np.float32)
381
+
382
+ # Supports:
383
+ # [x,y,w,h,score] single-class
384
+ # [x,y,w,h,obj,cls] YOLO standard single-class
385
+ # [x,y,w,h,obj,cls1,cls2,...] multi-class
386
+ if tail.shape[1] == 1:
387
+ scores = tail[:, 0]
388
+ cls_ids = np.zeros(len(scores), dtype=np.int32)
389
+ elif tail.shape[1] == 2:
390
+ obj = tail[:, 0]
391
+ cls_prob = tail[:, 1]
392
+ scores = obj * cls_prob
393
+ cls_ids = np.zeros(len(scores), dtype=np.int32)
394
+ else:
395
+ obj = tail[:, 0]
396
+ class_probs = tail[:, 1:]
397
+ cls_ids = np.argmax(class_probs, axis=1).astype(np.int32)
398
+ cls_scores = class_probs[np.arange(len(class_probs)), cls_ids]
399
+ scores = obj * cls_scores
400
+
401
+ keep = cls_ids == 0
402
+ boxes_xywh = boxes_xywh[keep]
403
+ scores = scores[keep]
404
+ cls_ids = cls_ids[keep]
405
+
406
+ keep = scores >= self.conf_thres
407
+ boxes_xywh = boxes_xywh[keep]
408
+ scores = scores[keep]
409
+ cls_ids = cls_ids[keep]
410
+
411
+ if len(boxes_xywh) == 0:
412
+ return []
413
+
414
+ boxes = self._xywh_to_xyxy(boxes_xywh)
415
+
416
+ pad_w, pad_h = pad
417
+ orig_w, orig_h = orig_size
418
+
419
+ boxes[:, [0, 2]] -= pad_w
420
+ boxes[:, [1, 3]] -= pad_h
421
+ boxes /= ratio
422
+ boxes = self._clip_boxes(boxes, (orig_w, orig_h))
423
+
424
+ boxes, scores, cls_ids = self._filter_sane_boxes(boxes, scores, cls_ids, orig_size)
425
+ if len(boxes) == 0:
426
+ return []
427
+
428
+ keep_idx = self._hard_nms(boxes, scores, self.iou_thres)
429
+ keep_idx = keep_idx[: self.max_det]
430
+
431
+ boxes = boxes[keep_idx]
432
+ scores = scores[keep_idx]
433
+ cls_ids = cls_ids[keep_idx]
434
+
435
+ return [
436
+ BoundingBox(
437
+ x1=int(math.floor(box[0])),
438
+ y1=int(math.floor(box[1])),
439
+ x2=int(math.ceil(box[2])),
440
+ y2=int(math.ceil(box[3])),
441
+ cls_id=int(cls_id),
442
+ conf=float(conf),
443
+ )
444
+ for box, conf, cls_id in zip(boxes, scores, cls_ids)
445
+ if box[2] > box[0] and box[3] > box[1]
446
+ ]
447
+
448
+ def _postprocess(
449
+ self,
450
+ output: np.ndarray,
451
+ ratio: float,
452
+ pad: tuple[float, float],
453
+ orig_size: tuple[int, int],
454
+ ) -> list[BoundingBox]:
455
+ if output.ndim == 2 and output.shape[1] >= 6:
456
+ return self._decode_final_dets(output, ratio, pad, orig_size)
457
+
458
+ if output.ndim == 3 and output.shape[0] == 1 and output.shape[2] >= 6:
459
+ return self._decode_final_dets(output, ratio, pad, orig_size)
460
+
461
+ return self._decode_raw_yolo(output, ratio, pad, orig_size)
462
+
463
+ def _predict_single(self, image: np.ndarray) -> list[BoundingBox]:
464
+ if image is None:
465
+ raise ValueError("Input image is None")
466
+ if not isinstance(image, np.ndarray):
467
+ raise TypeError(f"Input is not numpy array: {type(image)}")
468
+ if image.ndim != 3:
469
+ raise ValueError(f"Expected HWC image, got shape={image.shape}")
470
+ if image.shape[0] <= 0 or image.shape[1] <= 0:
471
+ raise ValueError(f"Invalid image shape={image.shape}")
472
+ if image.shape[2] != 3:
473
+ raise ValueError(f"Expected 3 channels, got shape={image.shape}")
474
+
475
+ if image.dtype != np.uint8:
476
+ image = image.astype(np.uint8)
477
+
478
+ input_tensor, ratio, pad, orig_size = self._preprocess(image)
479
+
480
+ expected_shape = (1, 3, self.input_height, self.input_width)
481
+ if input_tensor.shape != expected_shape:
482
+ raise ValueError(
483
+ f"Bad input tensor shape={input_tensor.shape}, expected={expected_shape}"
484
+ )
485
+
486
+ outputs = self.session.run(self.output_names, {self.input_name: input_tensor})
487
+ det_output = outputs[0]
488
+ return self._postprocess(det_output, ratio, pad, orig_size)
489
+
490
+ def _merge_tta_consensus(
491
+ self,
492
+ boxes_orig: list[BoundingBox],
493
+ boxes_flip: list[BoundingBox],
494
+ ) -> list[BoundingBox]:
495
+ """
496
+ Keep:
497
+ - any box with conf >= conf_high
498
+ - low/medium-conf boxes only if confirmed across TTA views
499
+ Then run final hard NMS.
500
+ """
501
+ if not boxes_orig and not boxes_flip:
502
+ return []
503
+
504
+ coords_o = np.array([[b.x1, b.y1, b.x2, b.y2] for b in boxes_orig], dtype=np.float32) if boxes_orig else np.empty((0, 4), dtype=np.float32)
505
+ scores_o = np.array([b.conf for b in boxes_orig], dtype=np.float32) if boxes_orig else np.empty((0,), dtype=np.float32)
506
+
507
+ coords_f = np.array([[b.x1, b.y1, b.x2, b.y2] for b in boxes_flip], dtype=np.float32) if boxes_flip else np.empty((0, 4), dtype=np.float32)
508
+ scores_f = np.array([b.conf for b in boxes_flip], dtype=np.float32) if boxes_flip else np.empty((0,), dtype=np.float32)
509
+
510
+ accepted_boxes = []
511
+ accepted_scores = []
512
+
513
+ # Original view candidates
514
+ for i in range(len(coords_o)):
515
+ score = scores_o[i]
516
+ if score >= self.conf_high:
517
+ accepted_boxes.append(coords_o[i])
518
+ accepted_scores.append(score)
519
+ elif len(coords_f) > 0:
520
+ ious = self._box_iou_one_to_many(coords_o[i], coords_f)
521
+ j = int(np.argmax(ious))
522
+ if ious[j] >= self.tta_match_iou:
523
+ fused_score = max(score, scores_f[j])
524
+ accepted_boxes.append(coords_o[i])
525
+ accepted_scores.append(fused_score)
526
+
527
+ # Flipped-view high-confidence boxes that original missed
528
+ for i in range(len(coords_f)):
529
+ score = scores_f[i]
530
+ if score < self.conf_high:
531
+ continue
532
+
533
+ if len(coords_o) == 0:
534
+ accepted_boxes.append(coords_f[i])
535
+ accepted_scores.append(score)
536
+ continue
537
+
538
+ ious = self._box_iou_one_to_many(coords_f[i], coords_o)
539
+ if np.max(ious) < self.tta_match_iou:
540
+ accepted_boxes.append(coords_f[i])
541
+ accepted_scores.append(score)
542
+
543
+ if not accepted_boxes:
544
+ return []
545
+
546
+ boxes = np.array(accepted_boxes, dtype=np.float32)
547
+ scores = np.array(accepted_scores, dtype=np.float32)
548
+
549
+ keep = self._hard_nms(boxes, scores, self.iou_thres)
550
+ keep = keep[: self.max_det]
551
+
552
+ out = []
553
+ for idx in keep:
554
+ x1, y1, x2, y2 = boxes[idx].tolist()
555
+ out.append(
556
+ BoundingBox(
557
+ x1=int(math.floor(x1)),
558
+ y1=int(math.floor(y1)),
559
+ x2=int(math.ceil(x2)),
560
+ y2=int(math.ceil(y2)),
561
+ cls_id=0,
562
+ conf=float(scores[idx]),
563
+ )
564
+ )
565
+ return out
566
+
567
+ def _predict_tta(self, image: np.ndarray) -> list[BoundingBox]:
568
+ boxes_orig = self._predict_single(image)
569
+
570
+ flipped = cv2.flip(image, 1)
571
+ boxes_flip_raw = self._predict_single(flipped)
572
+
573
+ w = image.shape[1]
574
+ boxes_flip = [
575
+ BoundingBox(
576
+ x1=w - b.x2,
577
+ y1=b.y1,
578
+ x2=w - b.x1,
579
+ y2=b.y2,
580
+ cls_id=b.cls_id,
581
+ conf=b.conf,
582
+ )
583
+ for b in boxes_flip_raw
584
+ ]
585
+
586
+ return self._merge_tta_consensus(boxes_orig, boxes_flip)
587
+
588
+ def _adaptive_conf_threshold(self, n_raw: int) -> float:
589
+ """
590
+ Dynamic confidence threshold based on raw detection count.
591
+
592
+ total_score = mAP50 * 0.65 + FP_score * 0.35
593
+ - Few objects → each TP worth ~0.065/n for mAP50 → keep low conf (maximize recall)
594
+ - Many objects → each TP worth little, FPs dominate → raise conf (minimize FP)
595
+ """
596
+ if n_raw <= self.count_low:
597
+ return self.conf_adapt_low
598
+ if n_raw >= self.count_high:
599
+ return self.conf_adapt_high
600
+ t = (n_raw - self.count_low) / (self.count_high - self.count_low)
601
+ return self.conf_adapt_low + t * (self.conf_adapt_high - self.conf_adapt_low)
602
+
603
+ def _apply_adaptive_filter(self, boxes: list[BoundingBox]) -> list[BoundingBox]:
604
+ if not boxes:
605
+ return boxes
606
+ n_raw = len(boxes)
607
+ thresh = self._adaptive_conf_threshold(n_raw)
608
+ return [b for b in boxes if b.conf >= thresh]
609
+
610
+ def predict_batch(
611
+ self,
612
+ batch_images: list[ndarray],
613
+ offset: int,
614
+ n_keypoints: int,
615
+ ) -> list[TVFrameResult]:
616
+ results: list[TVFrameResult] = []
617
+
618
+ for frame_number_in_batch, image in enumerate(batch_images):
619
+ try:
620
+ if self.use_tta:
621
+ boxes = self._predict_tta(image)
622
+ else:
623
+ boxes = self._predict_single(image)
624
+ boxes = self._apply_adaptive_filter(boxes)
625
+ except Exception as e:
626
+ print(f"⚠️ Inference failed for frame {offset + frame_number_in_batch}: {e}")
627
+ boxes = []
628
+
629
+ results.append(
630
+ TVFrameResult(
631
+ frame_id=offset + frame_number_in_batch,
632
+ boxes=boxes,
633
+ keypoints=[(0, 0) for _ in range(max(0, int(n_keypoints)))],
634
+ )
635
+ )
636
+
637
+ return results