coolroman commited on
Commit
4ec2b41
·
verified ·
1 Parent(s): a8f7f82

scorevision: push artifact

Browse files
Files changed (1) hide show
  1. miner.py +177 -475
miner.py CHANGED
@@ -1,22 +1,4 @@
1
- """TurboVision crime-detection miner.
2
-
3
- YOLO11s @ 1280x1280, 6-class detection (balaclava, bat, glove, graffiti, hoodie,
4
- spray paint), ONNX with end-to-end NMS baked in.
5
-
6
- Output of weights.onnx: [1, 300, 6] = x1, y1, x2, y2, conf, cls (post-NMS).
7
-
8
- Inference pipeline:
9
- 1) Primary forward pass on the full image.
10
- 2) Hflip TTA: forward on horizontally-flipped image, transform boxes back.
11
- 3) Per-class hard-NMS to merge primary + flip outputs.
12
- 4) Cross-class IoU dedup (suppresses same physical object getting two class labels).
13
- 5) Consensus-confidence boost: when both views agree on a cluster, take max score.
14
- 6) Sanity filter (min size, aspect ratio).
15
-
16
- Class taxonomy (must match the validator manifest's `objects` list for this element):
17
- 0 balaclava 1 bat 2 glove 3 graffiti 4 hoodie 5 spray paint
18
- """
19
-
20
  from pathlib import Path
21
  import math
22
 
@@ -43,43 +25,40 @@ class TVFrameResult(BaseModel):
43
 
44
 
45
  class Miner:
 
 
 
 
 
 
 
 
 
 
 
 
 
46
  def __init__(self, path_hf_repo: Path) -> None:
47
  model_path = path_hf_repo / "weights.onnx"
48
-
49
- # Validator manifest order (from spec.json `objects`):
50
- # 0=balaclava 1=hoodie 2=glove 3=bat 4="spray paint" 5=graffiti
51
- # v5 weights.onnx was trained with this exact order, so cls_remap is identity.
52
- cn_path = model_path.with_name("class_names.txt")
53
- if cn_path.is_file():
54
- self.class_names = [
55
- ln.strip()
56
- for ln in cn_path.read_text(encoding="utf-8").splitlines()
57
- if ln.strip() and not ln.strip().startswith("#")
58
- ]
59
- else:
60
- self.class_names = ["balaclava", "hoodie", "glove", "bat", "spray paint", "graffiti"]
61
- self.cls_remap = np.arange(len(self.class_names), dtype=np.int32)
62
-
63
  print("ORT version:", ort.__version__)
64
  try:
65
  ort.preload_dlls()
66
- print("✅ onnxruntime.preload_dlls() success")
67
  except Exception as e:
68
- print(f"⚠️ preload_dlls failed: {e}")
69
- print("ORT available providers BEFORE session:", ort.get_available_providers())
70
 
71
  sess_options = ort.SessionOptions()
72
  sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
73
-
74
  try:
75
  self.session = ort.InferenceSession(
76
  str(model_path),
77
  sess_options=sess_options,
78
  providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
79
  )
80
- print("Created ORT session with preferred CUDA provider list")
81
  except Exception as e:
82
- print(f"⚠️ CUDA session creation failed, falling back to CPU: {e}")
83
  self.session = ort.InferenceSession(
84
  str(model_path),
85
  sess_options=sess_options,
@@ -87,73 +66,46 @@ class Miner:
87
  )
88
  print("ORT session providers:", self.session.get_providers())
89
 
90
- inp = self.session.get_inputs()[0]
91
- self.input_name = inp.name
92
- self.output_names = [o.name for o in self.session.get_outputs()]
93
- self.input_shape = inp.shape
94
- self.input_dtype = np.float16 if "float16" in inp.type else np.float32
95
-
96
- self.input_height = self._safe_dim(self.input_shape[2], default=1280)
97
- self.input_width = self._safe_dim(self.input_shape[3], default=1280)
98
 
99
- # Tuning matched to alfred's deployed model — bias toward precision to dodge
100
- # the false_positive pillar penalty (validator weights FP heavily on this element).
101
- # v13 sweet spot on starter (true GT): uniform conf=0.50.
102
- # Tuning per-class on 7 images overfits — leave-one-out CV showed it
103
- # collapsed to 0.314 on held-out shards. Uniform 0.50 is robust.
104
- self.conf_thres = 0.50
105
- self.conf_thres_per_class = np.array([0.50] * 6, dtype=np.float32)
106
- self.iou_thres = 0.4
107
- self.cross_iou_thresh = 0.7
108
- self.max_det = 100
109
- self.use_tta = False
110
-
111
- # Sanity filter — reject obviously bad boxes
112
- self.min_box_area = 14 * 14
113
- self.min_side = 8
114
- self.max_aspect_ratio = 8.0
115
- self.max_box_area_ratio = 0.95
116
-
117
- print(f"✅ ONNX loaded: {model_path}")
118
- print(f"✅ providers: {self.session.get_providers()}")
119
- print(f"✅ input: name={self.input_name}, shape={self.input_shape}, dtype={self.input_dtype}")
120
- print(f"✅ classes: {self.class_names}")
121
- print(f"✅ config: conf={self.conf_thres}, iou={self.iou_thres}, "
122
- f"cross_iou={self.cross_iou_thresh}, TTA={self.use_tta}")
123
 
124
  def __repr__(self) -> str:
125
- return (
126
- f"ONNXRuntime(session={type(self.session).__name__}, "
127
- f"providers={self.session.get_providers()})"
128
- )
129
 
130
  @staticmethod
131
  def _safe_dim(value, default: int) -> int:
132
  return value if isinstance(value, int) and value > 0 else default
133
 
134
- def _letterbox(
135
- self,
136
- image: ndarray,
137
- new_shape: tuple[int, int],
138
- color=(114, 114, 114),
139
- ) -> tuple[ndarray, float, tuple[float, float]]:
140
  h, w = image.shape[:2]
141
  new_w, new_h = new_shape
142
  ratio = min(new_w / w, new_h / h)
143
- resized_w = int(round(w * ratio))
144
- resized_h = int(round(h * ratio))
145
- if (resized_w, resized_h) != (w, h):
146
  interp = cv2.INTER_CUBIC if ratio > 1.0 else cv2.INTER_LINEAR
147
- image = cv2.resize(image, (resized_w, resized_h), interpolation=interp)
148
- dw = (new_w - resized_w) / 2.0
149
- dh = (new_h - resized_h) / 2.0
150
- left = int(round(dw - 0.1))
151
- right = int(round(dw + 0.1))
152
- top = int(round(dh - 0.1))
153
- bottom = int(round(dh + 0.1))
154
  padded = cv2.copyMakeBorder(
155
- image, top, bottom, left, right,
156
- borderType=cv2.BORDER_CONSTANT, value=color,
157
  )
158
  return padded, ratio, (dw, dh)
159
 
@@ -161,445 +113,195 @@ class Miner:
161
  orig_h, orig_w = image.shape[:2]
162
  img, ratio, pad = self._letterbox(image, (self.input_width, self.input_height))
163
  img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
164
- img = img.astype(self.input_dtype) / 255.0
165
  img = np.transpose(img, (2, 0, 1))[None, ...]
166
- img = np.ascontiguousarray(img)
167
  return img, ratio, pad, (orig_w, orig_h)
168
 
169
  @staticmethod
170
- def _clip_boxes(boxes: np.ndarray, image_size: tuple[int, int]) -> np.ndarray:
171
- w, h = image_size
172
  boxes[:, 0] = np.clip(boxes[:, 0], 0, w - 1)
173
  boxes[:, 1] = np.clip(boxes[:, 1], 0, h - 1)
174
  boxes[:, 2] = np.clip(boxes[:, 2], 0, w - 1)
175
  boxes[:, 3] = np.clip(boxes[:, 3], 0, h - 1)
176
  return boxes
177
 
178
- def _filter_sane_boxes(
179
- self,
180
- boxes: np.ndarray,
181
- scores: np.ndarray,
182
- cls_ids: np.ndarray,
183
- orig_size: tuple[int, int],
184
- ):
185
- if len(boxes) == 0:
186
- return boxes, scores, cls_ids
187
- orig_w, orig_h = orig_size
188
- image_area = float(orig_w * orig_h)
189
  keep = []
190
- for i, box in enumerate(boxes):
191
- x1, y1, x2, y2 = box.tolist()
192
- bw = x2 - x1
193
- bh = y2 - y1
194
- if bw <= 0 or bh <= 0:
195
- continue
196
- if bw < self.min_side or bh < self.min_side:
197
- continue
198
- area = bw * bh
199
- if area < self.min_box_area:
200
- continue
201
- if area > self.max_box_area_ratio * image_area:
202
- continue
203
- ar = max(bw / max(bh, 1e-6), bh / max(bw, 1e-6))
204
- if ar > self.max_aspect_ratio:
205
- continue
206
  keep.append(i)
207
- if not keep:
208
- return (
209
- np.empty((0, 4), dtype=np.float32),
210
- np.empty((0,), dtype=np.float32),
211
- np.empty((0,), dtype=np.int32),
 
 
 
 
 
 
 
212
  )
213
- k = np.array(keep, dtype=np.intp)
214
- return boxes[k], scores[k], cls_ids[k]
215
-
216
- @staticmethod
217
- def _hard_nms(
218
- boxes: np.ndarray,
219
- scores: np.ndarray,
220
- iou_thresh: float,
221
- ) -> np.ndarray:
222
- N = len(boxes)
223
- if N == 0:
224
- return np.array([], dtype=np.intp)
225
- boxes = np.asarray(boxes, dtype=np.float32)
226
- scores = np.asarray(scores, dtype=np.float32)
227
- order = np.argsort(scores)[::-1]
228
- keep: list[int] = []
229
- suppressed = np.zeros(N, dtype=bool)
230
- for i in range(N):
231
- idx = order[i]
232
- if suppressed[idx]:
233
- continue
234
- keep.append(int(idx))
235
- bi = boxes[idx]
236
- for k in range(i + 1, N):
237
- jdx = order[k]
238
- if suppressed[jdx]:
239
- continue
240
- bj = boxes[jdx]
241
- xx1 = max(bi[0], bj[0])
242
- yy1 = max(bi[1], bj[1])
243
- xx2 = min(bi[2], bj[2])
244
- yy2 = min(bi[3], bj[3])
245
- inter = max(0.0, xx2 - xx1) * max(0.0, yy2 - yy1)
246
- area_i = (bi[2] - bi[0]) * (bi[3] - bi[1])
247
- area_j = (bj[2] - bj[0]) * (bj[3] - bj[1])
248
- iou = inter / (area_i + area_j - inter + 1e-7)
249
- if iou > iou_thresh:
250
- suppressed[jdx] = True
251
  return np.array(keep, dtype=np.intp)
252
 
253
- def _per_class_hard_nms(
254
- self,
255
- boxes: np.ndarray,
256
- scores: np.ndarray,
257
- cls_ids: np.ndarray,
258
- iou_thresh: float,
259
- ) -> np.ndarray:
260
  if len(boxes) == 0:
261
  return np.array([], dtype=np.intp)
262
- all_keep: list[int] = []
263
  for c in np.unique(cls_ids):
264
  mask = cls_ids == c
265
- indices = np.where(mask)[0]
266
- keep = self._hard_nms(boxes[mask], scores[mask], iou_thresh)
267
- all_keep.extend(indices[keep].tolist())
268
- all_keep.sort()
269
- return np.array(all_keep, dtype=np.intp)
270
-
271
- @staticmethod
272
- def _cross_class_dedup(
273
- boxes: np.ndarray,
274
- scores: np.ndarray,
275
- cls_ids: np.ndarray,
276
- iou_thresh: float,
277
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
278
  n = len(boxes)
279
- if n <= 1:
280
- return boxes, scores, cls_ids
281
- boxes = np.asarray(boxes, dtype=np.float32)
282
- scores = np.asarray(scores, dtype=np.float32)
283
- cls_ids = np.asarray(cls_ids, dtype=np.int32)
284
- areas = np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) * np.maximum(
285
- 0.0, boxes[:, 3] - boxes[:, 1]
286
- )
287
- # Keep larger boxes first, then higher score.
288
- order = np.lexsort((-scores, -areas))
289
  suppressed = np.zeros(n, dtype=bool)
290
- keep: list[int] = []
291
  for i in order:
292
  if suppressed[i]:
293
  continue
294
  keep.append(int(i))
295
- bi = boxes[i]
296
- xx1 = np.maximum(bi[0], boxes[:, 0])
297
- yy1 = np.maximum(bi[1], boxes[:, 1])
298
- xx2 = np.minimum(bi[2], boxes[:, 2])
299
- yy2 = np.minimum(bi[3], boxes[:, 3])
300
- inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
301
- area_i = max(1e-7, float((bi[2] - bi[0]) * (bi[3] - bi[1])))
302
- union = area_i + areas - inter + 1e-7
303
- iou = inter / union
304
- dup = iou > iou_thresh
305
- dup[i] = False
306
- suppressed |= dup
307
- keep_idx = np.array(keep, dtype=np.intp)
308
- return boxes[keep_idx], scores[keep_idx], cls_ids[keep_idx]
309
-
310
- @staticmethod
311
- def _max_score_per_cluster(
312
- coords: np.ndarray,
313
- scores: np.ndarray,
314
- keep_indices: np.ndarray,
315
- iou_thresh: float,
316
- ) -> np.ndarray:
317
- n_keep = len(keep_indices)
318
- if n_keep == 0:
319
- return np.array([], dtype=np.float32)
320
- coords = np.asarray(coords, dtype=np.float32)
321
- scores = np.asarray(scores, dtype=np.float32)
322
- out = np.empty(n_keep, dtype=np.float32)
323
- for i in range(n_keep):
324
- idx = keep_indices[i]
325
- bi = coords[idx]
326
- xx1 = np.maximum(bi[0], coords[:, 0])
327
- yy1 = np.maximum(bi[1], coords[:, 1])
328
- xx2 = np.minimum(bi[2], coords[:, 2])
329
- yy2 = np.minimum(bi[3], coords[:, 3])
330
- inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
331
- area_i = (bi[2] - bi[0]) * (bi[3] - bi[1])
332
- areas_j = (coords[:, 2] - coords[:, 0]) * (coords[:, 3] - coords[:, 1])
333
- iou = inter / (area_i + areas_j - inter + 1e-7)
334
- in_cluster = iou >= iou_thresh
335
- out[i] = float(np.max(scores[in_cluster]))
336
- return out
337
 
338
- def _decode_raw_dets(
339
- self,
340
- preds: np.ndarray,
341
- ratio: float,
342
- pad: tuple[float, float],
343
- orig_size: tuple[int, int],
344
- *,
345
- apply_conf_thresh: bool = True,
346
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
347
- """Decode end2end NMS output and return (boxes, scores, cls_ids)
348
- in original image coordinates, after conf-threshold + remap + letterbox-reverse + sanity.
 
 
 
 
349
 
350
- When apply_conf_thresh=False, the conf-threshold filter is skipped (used for
351
- the no-detection fallback path: take the single top-conf raw box)."""
 
352
  if preds.ndim == 3 and preds.shape[0] == 1:
353
  preds = preds[0]
354
  if preds.ndim != 2 or preds.shape[1] < 6:
355
- raise ValueError(f"Unexpected ONNX output shape: {preds.shape}")
356
-
357
- boxes = preds[:, :4].astype(np.float32)
358
- scores = preds[:, 4].astype(np.float32)
359
- cls_ids = preds[:, 5].astype(np.int32)
360
-
361
- valid = (cls_ids >= 0) & (cls_ids < len(self.cls_remap)) & (scores > 0)
362
- boxes, scores, cls_ids = boxes[valid], scores[valid], cls_ids[valid]
363
- cls_ids = self.cls_remap[cls_ids]
364
-
365
- if apply_conf_thresh:
366
- # Per-class threshold: each box compared against its own class's threshold
367
- cls_thresh = np.full(len(scores), self.conf_thres, dtype=np.float32)
368
- valid_cls = (cls_ids >= 0) & (cls_ids < len(self.conf_thres_per_class))
369
- cls_thresh[valid_cls] = self.conf_thres_per_class[cls_ids[valid_cls]]
370
- keep = scores >= cls_thresh
371
- boxes = boxes[keep]
372
- scores = scores[keep]
373
- cls_ids = cls_ids[keep]
374
- if len(boxes) == 0:
375
  return (
376
  np.empty((0, 4), dtype=np.float32),
377
  np.empty((0,), dtype=np.float32),
378
  np.empty((0,), dtype=np.int32),
379
  )
380
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
381
  pad_w, pad_h = pad
382
- orig_w, orig_h = orig_size
383
  boxes[:, [0, 2]] -= pad_w
384
  boxes[:, [1, 3]] -= pad_h
385
  boxes /= ratio
386
- boxes = self._clip_boxes(boxes, (orig_w, orig_h))
387
-
388
- boxes, scores, cls_ids = self._filter_sane_boxes(boxes, scores, cls_ids, orig_size)
389
  return boxes, scores, cls_ids
390
 
391
- def _forward(
392
- self, image: np.ndarray
393
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
394
  x, ratio, pad, orig_size = self._preprocess(image)
395
  out = self.session.run(self.output_names, {self.input_name: x})[0]
396
- return self._decode_raw_dets(out, ratio, pad, orig_size)
397
 
398
- def _forward_with_fallback(
399
- self, image: np.ndarray
400
- ) -> tuple[
401
- tuple[np.ndarray, np.ndarray, np.ndarray],
402
- tuple[np.ndarray, np.ndarray, np.ndarray],
403
- ]:
404
- """Run ONNX once, decode twice: (filtered @ conf_thres, all-survived sanity)."""
405
- x, ratio, pad, orig_size = self._preprocess(image)
406
- out = self.session.run(self.output_names, {self.input_name: x})[0]
407
- primary = self._decode_raw_dets(out, ratio, pad, orig_size, apply_conf_thresh=True)
408
- fallback = self._decode_raw_dets(out, ratio, pad, orig_size, apply_conf_thresh=False)
409
- return primary, fallback
410
-
411
- def _predict_single(self, image: np.ndarray) -> list[BoundingBox]:
412
- (boxes, scores, cls_ids), (fb_b, fb_s, fb_c) = self._forward_with_fallback(image)
413
- ih, iw = image.shape[:2]
414
- if len(boxes) > 0:
415
- return self._build_results(boxes, scores, cls_ids, image_size=(iw, ih))
416
- # FALLBACK: nothing passed conf_thres — return single top-conf box
417
- # (any class, any conf > 0) so the validator's mAP isn't a hard zero.
418
- if len(fb_b) == 0:
419
  return []
420
- i = int(np.argmax(fb_s))
421
- return self._build_results(
422
- fb_b[i:i + 1], fb_s[i:i + 1], fb_c[i:i + 1], image_size=(iw, ih)
423
  )
424
-
425
- def _predict_tta(self, image: np.ndarray) -> list[BoundingBox]:
426
- """Hflip TTA: merge primary + flipped via per-class hard-NMS,
427
- then cross-class dedup, with consensus-confidence boost."""
428
- ow = image.shape[1]
429
- b1, s1, c1 = self._forward(image)
430
-
431
- flipped = cv2.flip(image, 1)
432
- b2, s2, c2 = self._forward(flipped)
433
- if len(b2):
434
- x1f = ow - b2[:, 2]
435
- x2f = ow - b2[:, 0]
436
- b2 = np.stack([x1f, b2[:, 1], x2f, b2[:, 3]], axis=1)
437
-
438
- if len(b1) == 0 and len(b2) == 0:
439
  return []
440
-
441
- boxes = np.concatenate([b1, b2], axis=0) if len(b2) else b1
442
- scores = np.concatenate([s1, s2], axis=0) if len(b2) else s1
443
- cls_ids = np.concatenate([c1, c2], axis=0) if len(b2) else c1
444
-
445
  keep = self._per_class_hard_nms(boxes, scores, cls_ids, self.iou_thres)
446
  if len(keep) == 0:
447
  return []
448
- keep = keep[: self.max_det]
449
-
450
- # Consensus-confidence boost: cluster by IoU and take max score.
451
- boosted = self._max_score_per_cluster(boxes, scores, keep, self.iou_thres)
452
-
453
- boxes = boxes[keep]
454
- cls_ids = cls_ids[keep]
455
- scores = boosted
456
-
457
- boxes, scores, cls_ids = self._cross_class_dedup(
458
- boxes, scores, cls_ids, self.cross_iou_thresh
459
- )
460
- if len(boxes) == 0:
461
- return []
462
-
463
- ih, iw = image.shape[:2]
464
- return self._build_results(boxes, scores, cls_ids, image_size=(iw, ih))
465
-
466
- def _filter_balaclava_geometry(
467
- self,
468
- boxes: np.ndarray,
469
- scores: np.ndarray,
470
- cls_ids: np.ndarray,
471
- image_size: tuple[int, int] | None = None,
472
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
473
- # Real-balaclava prior (from 43 manual GT labels):
474
- # aspect ratio max(w/h, h/w): p5=1.11, median=1.33, p99=1.71
475
- # rel area % of image: p1=0.041, p5=0.070, p10=0.087
476
- # FP balaclavas frequently violate these (very thin/wide boxes from
477
- # face-fragment matches, or tiny ~0.01%-area boxes from texture noise).
478
- BALACLAVA = 0
479
- ASPECT_MAX = 1.8 # above p99 of real
480
- REL_AREA_MIN = 0.0004 # below p1 of real (0.04%)
481
- if len(boxes) == 0:
482
- return boxes, scores, cls_ids
483
- is_bal = cls_ids == BALACLAVA
484
- if not is_bal.any():
485
- return boxes, scores, cls_ids
486
- keep = np.ones(len(boxes), dtype=bool)
487
- if image_size is not None:
488
- iw, ih = image_size
489
- img_area = max(1.0, iw * ih)
490
- else:
491
- img_area = None
492
- for i in np.where(is_bal)[0]:
493
- x1, y1, x2, y2 = boxes[i]
494
- bw = max(1.0, x2 - x1)
495
- bh = max(1.0, y2 - y1)
496
- aspect = max(bw / bh, bh / bw)
497
- if aspect > ASPECT_MAX:
498
- keep[i] = False
499
- continue
500
- if img_area is not None:
501
- rel = (bw * bh) / img_area
502
- if rel < REL_AREA_MIN:
503
- keep[i] = False
504
- return boxes[keep], scores[keep], cls_ids[keep]
505
-
506
- def _suppress_balaclava_under_hoodie(
507
- self,
508
- boxes: np.ndarray,
509
- scores: np.ndarray,
510
- cls_ids: np.ndarray,
511
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
512
- # Validator rule: "balaclavas worn under a hoodie hood are IGNORED
513
- # (a hoodie includes the jacket and its hood)". A small balaclava
514
- # box can sit fully inside a much larger hoodie box — IoU between
515
- # them stays low (intersection / large union), but containment
516
- # (intersection / balaclava_area) is ~1.0. So drop any balaclava
517
- # whose containment by any hoodie box is >= COVER_THRESH.
518
- BALACLAVA, HOODIE = 0, 1
519
- COVER_THRESH = 0.5
520
- if len(boxes) == 0:
521
- return boxes, scores, cls_ids
522
- is_hood = cls_ids == HOODIE
523
- is_bal = cls_ids == BALACLAVA
524
- if not is_hood.any() or not is_bal.any():
525
- return boxes, scores, cls_ids
526
- hood_boxes = boxes[is_hood]
527
- keep = np.ones(len(boxes), dtype=bool)
528
- for i in np.where(is_bal)[0]:
529
- bx1, by1, bx2, by2 = boxes[i]
530
- bal_area = max(1.0, (bx2 - bx1) * (by2 - by1))
531
- ix1 = np.maximum(bx1, hood_boxes[:, 0])
532
- iy1 = np.maximum(by1, hood_boxes[:, 1])
533
- ix2 = np.minimum(bx2, hood_boxes[:, 2])
534
- iy2 = np.minimum(by2, hood_boxes[:, 3])
535
- iw = np.clip(ix2 - ix1, 0.0, None)
536
- ih = np.clip(iy2 - iy1, 0.0, None)
537
- inter = iw * ih
538
- cover = inter / bal_area
539
- if (cover >= COVER_THRESH).any():
540
- keep[i] = False
541
- return boxes[keep], scores[keep], cls_ids[keep]
542
-
543
- def _build_results(
544
- self,
545
- boxes: np.ndarray,
546
- scores: np.ndarray,
547
- cls_ids: np.ndarray,
548
- image_size: tuple[int, int] | None = None,
549
- ) -> list[BoundingBox]:
550
- boxes, scores, cls_ids = self._filter_balaclava_geometry(
551
- boxes, scores, cls_ids, image_size
552
- )
553
- boxes, scores, cls_ids = self._suppress_balaclava_under_hoodie(
554
- boxes, scores, cls_ids
555
- )
556
- results: list[BoundingBox] = []
557
- for box, conf, cls_id in zip(boxes, scores, cls_ids):
558
- x1, y1, x2, y2 = box.tolist()
559
- if x2 <= x1 or y2 <= y1:
560
- continue
561
- results.append(
562
- BoundingBox(
563
- x1=int(math.floor(x1)),
564
- y1=int(math.floor(y1)),
565
- x2=int(math.ceil(x2)),
566
- y2=int(math.ceil(y2)),
567
- cls_id=int(cls_id),
568
- conf=float(conf),
569
- )
570
  )
571
- return results
 
 
572
 
573
- def predict_batch(
574
- self,
575
- batch_images: list[ndarray],
576
- offset: int,
577
- n_keypoints: int,
578
- ) -> list[TVFrameResult]:
579
  results: list[TVFrameResult] = []
580
- for frame_number_in_batch, image in enumerate(batch_images):
581
- if image is None or not isinstance(image, np.ndarray) or image.ndim != 3:
582
- results.append(
583
- TVFrameResult(
584
- frame_id=offset + frame_number_in_batch,
585
- boxes=[],
586
- keypoints=[(0, 0) for _ in range(max(0, int(n_keypoints)))],
587
- )
588
- )
589
- continue
590
- if image.dtype != np.uint8:
591
- image = image.astype(np.uint8)
592
  try:
593
- if self.use_tta:
594
- boxes = self._predict_tta(image)
595
- else:
596
- boxes = self._predict_single(image)
597
  except Exception as e:
598
- print(f"⚠️ Inference failed for frame {offset + frame_number_in_batch}: {e}")
599
  boxes = []
600
  results.append(
601
  TVFrameResult(
602
- frame_id=offset + frame_number_in_batch,
603
  boxes=boxes,
604
  keypoints=[(0, 0) for _ in range(max(0, int(n_keypoints)))],
605
  )
 
1
+ """ScoreVision crime detector — YOLOv11s with flip TTA + per-class conf + cross-class NMS."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  from pathlib import Path
3
  import math
4
 
 
25
 
26
 
27
  class Miner:
28
+ """ONNX Runtime miner with horizontal-flip TTA and per-class confidence."""
29
+
30
+ class_names = ["balaclava", "hoodie", "glove", "bat", "spray paint", "graffiti"]
31
+ input_size = 1280
32
+ iou_thres = 0.45
33
+ cross_iou_thresh = 0.50
34
+ max_aspect_ratio = 10.0
35
+ max_det = 150
36
+ # tuned on held-out SAM3-labeled crime set
37
+ _conf_thres_array = np.array(
38
+ [0.50, 0.50, 0.30, 0.30, 0.40, 0.40], dtype=np.float32
39
+ )
40
+
41
  def __init__(self, path_hf_repo: Path) -> None:
42
  model_path = path_hf_repo / "weights.onnx"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
  print("ORT version:", ort.__version__)
44
  try:
45
  ort.preload_dlls()
46
+ print("preload_dlls success")
47
  except Exception as e:
48
+ print(f"preload_dlls failed: {e}")
49
+ print("ORT available providers:", ort.get_available_providers())
50
 
51
  sess_options = ort.SessionOptions()
52
  sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
 
53
  try:
54
  self.session = ort.InferenceSession(
55
  str(model_path),
56
  sess_options=sess_options,
57
  providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
58
  )
59
+ print("Created ORT session with preferred CUDA provider list")
60
  except Exception as e:
61
+ print(f"CUDA session creation failed, falling back to CPU: {e}")
62
  self.session = ort.InferenceSession(
63
  str(model_path),
64
  sess_options=sess_options,
 
66
  )
67
  print("ORT session providers:", self.session.get_providers())
68
 
69
+ for inp in self.session.get_inputs():
70
+ print("INPUT:", inp.name, inp.shape, inp.type)
71
+ for out in self.session.get_outputs():
72
+ print("OUTPUT:", out.name, out.shape, out.type)
 
 
 
 
73
 
74
+ self.input_name = self.session.get_inputs()[0].name
75
+ self.output_names = [o.name for o in self.session.get_outputs()]
76
+ self.input_shape = self.session.get_inputs()[0].shape
77
+ self.input_height = self._safe_dim(self.input_shape[2], self.input_size)
78
+ self.input_width = self._safe_dim(self.input_shape[3], self.input_size)
79
+ print(f"ONNX loaded: {model_path} input={self.input_shape}")
80
+ print(
81
+ "per-class conf: "
82
+ + ", ".join(
83
+ f"{n}={t:.3f}" for n, t in zip(self.class_names, self._conf_thres_array.tolist())
84
+ )
85
+ )
 
 
 
 
 
 
 
 
 
 
 
 
86
 
87
  def __repr__(self) -> str:
88
+ return f"ONNXRuntime(providers={self.session.get_providers()})"
 
 
 
89
 
90
  @staticmethod
91
  def _safe_dim(value, default: int) -> int:
92
  return value if isinstance(value, int) and value > 0 else default
93
 
94
+ def _letterbox(self, image: ndarray, new_shape: tuple[int, int],
95
+ color=(114, 114, 114)
96
+ ) -> tuple[ndarray, float, tuple[float, float]]:
 
 
 
97
  h, w = image.shape[:2]
98
  new_w, new_h = new_shape
99
  ratio = min(new_w / w, new_h / h)
100
+ rw, rh = int(round(w * ratio)), int(round(h * ratio))
101
+ if (rw, rh) != (w, h):
 
102
  interp = cv2.INTER_CUBIC if ratio > 1.0 else cv2.INTER_LINEAR
103
+ image = cv2.resize(image, (rw, rh), interpolation=interp)
104
+ dw, dh = (new_w - rw) / 2.0, (new_h - rh) / 2.0
105
+ top, bot = int(round(dh - 0.1)), int(round(dh + 0.1))
106
+ lt, rt = int(round(dw - 0.1)), int(round(dw + 0.1))
 
 
 
107
  padded = cv2.copyMakeBorder(
108
+ image, top, bot, lt, rt, cv2.BORDER_CONSTANT, value=color
 
109
  )
110
  return padded, ratio, (dw, dh)
111
 
 
113
  orig_h, orig_w = image.shape[:2]
114
  img, ratio, pad = self._letterbox(image, (self.input_width, self.input_height))
115
  img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
116
+ img = img.astype(np.float32) / 255.0
117
  img = np.transpose(img, (2, 0, 1))[None, ...]
118
+ img = np.ascontiguousarray(img, dtype=np.float32)
119
  return img, ratio, pad, (orig_w, orig_h)
120
 
121
  @staticmethod
122
+ def _clip(boxes, size):
123
+ w, h = size
124
  boxes[:, 0] = np.clip(boxes[:, 0], 0, w - 1)
125
  boxes[:, 1] = np.clip(boxes[:, 1], 0, h - 1)
126
  boxes[:, 2] = np.clip(boxes[:, 2], 0, w - 1)
127
  boxes[:, 3] = np.clip(boxes[:, 3], 0, h - 1)
128
  return boxes
129
 
130
+ @staticmethod
131
+ def _hard_nms(boxes, scores, iou_thr):
132
+ n = len(boxes)
133
+ if n == 0:
134
+ return np.array([], dtype=np.intp)
135
+ order = np.argsort(-scores)
 
 
 
 
 
136
  keep = []
137
+ while len(order) > 0:
138
+ i = int(order[0])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
139
  keep.append(i)
140
+ if len(order) == 1:
141
+ break
142
+ rest = order[1:]
143
+ xx1 = np.maximum(boxes[i, 0], boxes[rest, 0])
144
+ yy1 = np.maximum(boxes[i, 1], boxes[rest, 1])
145
+ xx2 = np.minimum(boxes[i, 2], boxes[rest, 2])
146
+ yy2 = np.minimum(boxes[i, 3], boxes[rest, 3])
147
+ inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
148
+ a_i = max(0.0, boxes[i, 2] - boxes[i, 0]) * max(0.0, boxes[i, 3] - boxes[i, 1])
149
+ a_r = (
150
+ np.maximum(0.0, boxes[rest, 2] - boxes[rest, 0])
151
+ * np.maximum(0.0, boxes[rest, 3] - boxes[rest, 1])
152
  )
153
+ iou = inter / (a_i + a_r - inter + 1e-7)
154
+ order = rest[iou <= iou_thr]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
155
  return np.array(keep, dtype=np.intp)
156
 
157
+ def _per_class_hard_nms(self, boxes, scores, cls_ids, iou_thr):
 
 
 
 
 
 
158
  if len(boxes) == 0:
159
  return np.array([], dtype=np.intp)
160
+ keep_all = []
161
  for c in np.unique(cls_ids):
162
  mask = cls_ids == c
163
+ idx = np.where(mask)[0]
164
+ keep = self._hard_nms(boxes[mask], scores[mask], iou_thr)
165
+ keep_all.extend(idx[keep].tolist())
166
+ keep_all.sort()
167
+ return np.array(keep_all, dtype=np.intp)
168
+
169
+ def _cross_class_dedup(self, boxes, scores, cls_ids, iou_thr):
170
+ """If two boxes (any class) heavily overlap, keep only the higher-scoring one."""
 
 
 
 
 
171
  n = len(boxes)
172
+ if n == 0:
173
+ return np.array([], dtype=np.intp)
174
+ order = np.argsort(-scores)
175
+ keep = []
 
 
 
 
 
 
176
  suppressed = np.zeros(n, dtype=bool)
 
177
  for i in order:
178
  if suppressed[i]:
179
  continue
180
  keep.append(int(i))
181
+ ix1 = np.maximum(boxes[i, 0], boxes[:, 0])
182
+ iy1 = np.maximum(boxes[i, 1], boxes[:, 1])
183
+ ix2 = np.minimum(boxes[i, 2], boxes[:, 2])
184
+ iy2 = np.minimum(boxes[i, 3], boxes[:, 3])
185
+ inter = np.maximum(0.0, ix2 - ix1) * np.maximum(0.0, iy2 - iy1)
186
+ a_i = max(0.0, boxes[i, 2] - boxes[i, 0]) * max(0.0, boxes[i, 3] - boxes[i, 1])
187
+ a_r = (
188
+ np.maximum(0.0, boxes[:, 2] - boxes[:, 0])
189
+ * np.maximum(0.0, boxes[:, 3] - boxes[:, 1])
190
+ )
191
+ iou = inter / (a_i + a_r - inter + 1e-7)
192
+ iou[i] = 0.0
193
+ suppressed |= iou >= iou_thr
194
+ return np.array(keep, dtype=np.intp)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
195
 
196
+ def _filter_sane(self, boxes, scores, cls_ids, orig_size):
197
+ if len(boxes) == 0:
198
+ return boxes, scores, cls_ids
199
+ w, h = orig_size
200
+ area_img = float(w * h)
201
+ bw = np.maximum(0.0, boxes[:, 2] - boxes[:, 0])
202
+ bh = np.maximum(0.0, boxes[:, 3] - boxes[:, 1])
203
+ area = bw * bh
204
+ ar = np.where(
205
+ (bw > 0) & (bh > 0),
206
+ np.maximum(bw / np.maximum(bh, 1e-6), bh / np.maximum(bw, 1e-6)),
207
+ np.inf,
208
+ )
209
+ keep = (area <= 0.95 * area_img) & (ar <= self.max_aspect_ratio)
210
+ return boxes[keep], scores[keep], cls_ids[keep]
211
 
212
+ def _decode(self, preds: ndarray, ratio: float, pad: tuple[float, float],
213
+ orig_size: tuple[int, int]) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
214
+ """ONNX output is [1,300,6]: x1,y1,x2,y2,conf,cls in letterboxed coords."""
215
  if preds.ndim == 3 and preds.shape[0] == 1:
216
  preds = preds[0]
217
  if preds.ndim != 2 or preds.shape[1] < 6:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
218
  return (
219
  np.empty((0, 4), dtype=np.float32),
220
  np.empty((0,), dtype=np.float32),
221
  np.empty((0,), dtype=np.int32),
222
  )
223
+ boxes = preds[:, :4].astype(np.float32)
224
+ scores = preds[:, 4].astype(np.float32)
225
+ cls_ids = preds[:, 5].astype(np.int32)
226
+ # drop padded rows
227
+ keep = (scores > 0) & (cls_ids >= 0) & (cls_ids < len(self.class_names))
228
+ boxes, scores, cls_ids = boxes[keep], scores[keep], cls_ids[keep]
229
+ if len(boxes) == 0:
230
+ return boxes, scores, cls_ids
231
+ # per-class confidence
232
+ thr = self._conf_thres_array[cls_ids]
233
+ keep = scores >= thr
234
+ boxes, scores, cls_ids = boxes[keep], scores[keep], cls_ids[keep]
235
+ if len(boxes) == 0:
236
+ return boxes, scores, cls_ids
237
+ # untransform: subtract pad, divide ratio
238
  pad_w, pad_h = pad
 
239
  boxes[:, [0, 2]] -= pad_w
240
  boxes[:, [1, 3]] -= pad_h
241
  boxes /= ratio
242
+ boxes = self._clip(boxes, orig_size)
 
 
243
  return boxes, scores, cls_ids
244
 
245
+ def _predict_single(self, image: ndarray):
 
 
246
  x, ratio, pad, orig_size = self._preprocess(image)
247
  out = self.session.run(self.output_names, {self.input_name: x})[0]
248
+ return self._decode(out, ratio, pad, orig_size)
249
 
250
+ def _predict_tta(self, image: ndarray) -> list[BoundingBox]:
251
+ b0, s0, c0 = self._predict_single(image)
252
+ flipped = cv2.flip(image, 1)
253
+ bf, sf, cf = self._predict_single(flipped)
254
+ if len(bf):
255
+ w = image.shape[1]
256
+ bf2 = bf.copy()
257
+ bf2[:, 0] = w - bf[:, 2]
258
+ bf2[:, 2] = w - bf[:, 0]
259
+ bf = bf2
260
+ boxes = np.concatenate([b0, bf], axis=0) if len(b0) or len(bf) else b0
261
+ scores = np.concatenate([s0, sf], axis=0) if len(b0) or len(bf) else s0
262
+ cls_ids = np.concatenate([c0, cf], axis=0) if len(b0) or len(bf) else c0
263
+ if len(boxes) == 0:
 
 
 
 
 
 
 
264
  return []
265
+ # filter + per-class NMS + cross-class dedup
266
+ boxes, scores, cls_ids = self._filter_sane(
267
+ boxes, scores, cls_ids, (image.shape[1], image.shape[0])
268
  )
269
+ if len(boxes) == 0:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
270
  return []
 
 
 
 
 
271
  keep = self._per_class_hard_nms(boxes, scores, cls_ids, self.iou_thres)
272
  if len(keep) == 0:
273
  return []
274
+ boxes, scores, cls_ids = boxes[keep], scores[keep], cls_ids[keep]
275
+ keep = self._cross_class_dedup(boxes, scores, cls_ids, self.cross_iou_thresh)
276
+ boxes, scores, cls_ids = boxes[keep], scores[keep], cls_ids[keep]
277
+ if len(scores) > self.max_det:
278
+ top = np.argsort(-scores)[: self.max_det]
279
+ boxes, scores, cls_ids = boxes[top], scores[top], cls_ids[top]
280
+ return [
281
+ BoundingBox(
282
+ x1=int(math.floor(b[0])),
283
+ y1=int(math.floor(b[1])),
284
+ x2=int(math.ceil(b[2])),
285
+ y2=int(math.ceil(b[3])),
286
+ cls_id=int(c),
287
+ conf=float(s),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
288
  )
289
+ for b, s, c in zip(boxes, scores, cls_ids)
290
+ if b[2] > b[0] and b[3] > b[1]
291
+ ]
292
 
293
+ def predict_batch(self, batch_images: list[ndarray], offset: int,
294
+ n_keypoints: int) -> list[TVFrameResult]:
 
 
 
 
295
  results: list[TVFrameResult] = []
296
+ for j, image in enumerate(batch_images):
 
 
 
 
 
 
 
 
 
 
 
297
  try:
298
+ boxes = self._predict_tta(image)
 
 
 
299
  except Exception as e:
300
+ print(f"Inference failed for frame {offset + j}: {e}")
301
  boxes = []
302
  results.append(
303
  TVFrameResult(
304
+ frame_id=offset + j,
305
  boxes=boxes,
306
  keypoints=[(0, 0) for _ in range(max(0, int(n_keypoints)))],
307
  )