SuperBitDev commited on
Commit
ba3e615
·
verified ·
1 Parent(s): 95b1912

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. miner.py +689 -420
  2. weights.onnx +2 -2
miner.py CHANGED
@@ -24,141 +24,254 @@ class TVFrameResult(BaseModel):
24
 
25
 
26
  class Miner:
27
- """
28
- YOLOv26 ONNX miner for car wash detection.
29
-
30
- Classes: broom, drainage gate, nozzle, track
31
-
32
- v26 is NMS-free — output shape: [1, 300, 6] (x1, y1, x2, y2, conf, cls_id).
33
-
34
- Features:
35
- - Vectorized NMS + sanity filter + dedup + flip TTA
36
- - Per-class rescue bonus (saves hard-to-detect classes at slightly lower conf)
37
- - Confidence boost from same-class cluster (TTA consensus)
38
- - Aggressive same-class overlap suppression
39
- - Per-class IoU thresholds
40
- """
41
-
42
- class_names = ['broom', 'drainage gate', 'nozzle', 'track']
43
- input_size = 1408
44
- cross_iou_thresh = 0.8
45
- max_det = 300
46
- #overlap_suppress_threshold = 0.85
47
-
48
- # Per-class confidence thresholds
49
- _conf_thres_array = np.array([0.35, 0.7, 0.4, 0.7], dtype=np.float32)
50
- _extra_conf_thres_array = np.array([0.35, 0.35, 0.55, 0.35], dtype=np.float32)
51
-
52
- # Per-class IoU thresholds for same-class NMS
53
- _iou_thres_array = np.array([0.6, 0.65, 0.5, 0.65], dtype=np.float32)
54
-
55
- # Per-class rescue bonus
56
- _bonus_array = np.array([0.2, 0.2, 0.0, 0.2], dtype=np.float32)
57
-
58
- # Per-class minimum box area
59
- # Indices: 0=broom, 1=drainage gate, 2=nozzle, 3=track
60
- _min_box_area_array = np.array([144.0, 144.0, 4.0, 144.0], dtype=np.float32)
61
-
62
-
63
- def __init__(self, path_hf_repo: Path) -> None:
64
- self.path_hf_repo = path_hf_repo
65
-
66
  print("ORT version:", ort.__version__)
67
-
68
  try:
69
  ort.preload_dlls()
70
- print("preload_dlls success")
71
  except Exception as e:
72
- print(f"preload_dlls failed: {e}")
73
-
74
  print("ORT available providers BEFORE session:", ort.get_available_providers())
75
-
76
  sess_options = ort.SessionOptions()
77
  sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
78
-
79
- self.session = ort.InferenceSession(
80
- str(path_hf_repo / "weights.onnx"),
81
- sess_options=sess_options,
82
- providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
83
- )
84
- print("Created ORT session with preferred CUDA provider list")
 
 
 
 
 
 
 
 
 
 
 
 
85
  print("ORT session providers:", self.session.get_providers())
86
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
  self.input_name = self.session.get_inputs()[0].name
88
  self.output_names = [output.name for output in self.session.get_outputs()]
89
- input_shape = self.session.get_inputs()[0].shape
90
-
91
- self.input_h = self._safe_dim(input_shape[2], default=self.input_size)
92
- self.input_w = self._safe_dim(input_shape[3], default=self.input_size)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93
 
94
  def __repr__(self) -> str:
95
- return f"YOLOv26 Car Wash Miner classes={len(self.class_names)}"
 
 
 
96
 
97
  @staticmethod
98
  def _safe_dim(value, default: int) -> int:
99
  return value if isinstance(value, int) and value > 0 else default
100
 
101
- # ─── Preprocessing ────────────────────────────────────────────
102
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
  def _letterbox(
104
- self, image: ndarray, new_shape: tuple[int, int],
105
- color: tuple[int, int, int] = (114, 114, 114),
106
- ) -> tuple[ndarray, float, float, float]:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
107
  orig_h, orig_w = image.shape[:2]
108
- target_w, target_h = new_shape
109
-
110
- r = min(target_w / orig_w, target_h / orig_h)
111
- new_unpad_w = int(round(orig_w * r))
112
- new_unpad_h = int(round(orig_h * r))
113
-
114
- resized = cv2.resize(image, (new_unpad_w, new_unpad_h), interpolation=cv2.INTER_LINEAR)
115
-
116
- dw = target_w - new_unpad_w
117
- dh = target_h - new_unpad_h
118
- pad_w = dw / 2.0
119
- pad_h = dh / 2.0
120
-
121
- left = int(round(pad_w - 0.1))
122
- right = int(round(pad_w + 0.1))
123
- top = int(round(pad_h - 0.1))
124
- bottom = int(round(pad_h + 0.1))
125
-
126
- out = cv2.copyMakeBorder(
127
- resized, top, bottom, left, right,
128
- cv2.BORDER_CONSTANT, value=color,
129
  )
130
- return out, r, pad_w, pad_h
131
-
132
- def _preprocess(self, image_bgr: np.ndarray,
133
- allow_pad: bool = True) -> tuple[np.ndarray, dict]:
134
- orig_h, orig_w = image_bgr.shape[:2]
135
- extra_left = 0
136
- extra_right = 0
137
- if allow_pad and orig_w == orig_h: # only pad when allowed
138
- target_w = int(orig_w * 1.05)
139
- if target_w > orig_w:
140
- total_extra = target_w - orig_w
141
- extra_left = total_extra // 2
142
- extra_right = total_extra - extra_left
143
- image_bgr = cv2.copyMakeBorder(
144
- image_bgr, 0, 0, extra_left, extra_right,
145
- cv2.BORDER_CONSTANT, value=(114, 114, 114),
146
- )
147
- padded_h, padded_w = image_bgr.shape[:2]
148
- rgb = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)
149
- img, ratio, pad_w, pad_h = self._letterbox(rgb, (self.input_w, self.input_h))
150
- x = img.astype(np.float32) / 255.0
151
- x = np.transpose(x, (2, 0, 1))[None, ...]
152
- x = np.ascontiguousarray(x)
153
- return x, {
154
- "orig_h": orig_h, "orig_w": orig_w,
155
- "ratio": ratio, "pad_w": pad_w, "pad_h": pad_h,
156
- "extra_left": extra_left, "extra_right": extra_right,
157
- "padded_w": padded_w, "padded_h": padded_h,
158
- }
159
-
160
- # ─── Vectorized box operations ───────────────────────────────
161
-
162
  @staticmethod
163
  def _clip_boxes(boxes: np.ndarray, image_size: tuple[int, int]) -> np.ndarray:
164
  w, h = image_size
@@ -169,116 +282,200 @@ class Miner:
169
  return boxes
170
 
171
  @staticmethod
172
- def _hard_nms(boxes: np.ndarray, scores: np.ndarray,
173
- iou_thresh: float) -> np.ndarray:
174
- """Vectorized NMS. Returns indices to keep."""
175
- n = len(boxes)
176
- if n == 0:
177
- return np.array([], dtype=np.intp)
178
- order = np.argsort(-scores)
179
- keep = []
180
- while len(order) > 0:
181
- i = int(order[0])
182
- keep.append(i)
183
- if len(order) == 1:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
184
  break
185
- rest = order[1:]
186
- xx1 = np.maximum(boxes[i, 0], boxes[rest, 0])
187
- yy1 = np.maximum(boxes[i, 1], boxes[rest, 1])
188
- xx2 = np.minimum(boxes[i, 2], boxes[rest, 2])
189
- yy2 = np.minimum(boxes[i, 3], boxes[rest, 3])
190
  inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
191
- a_i = (max(0.0, boxes[i, 2] - boxes[i, 0]) *
192
- max(0.0, boxes[i, 3] - boxes[i, 1]))
193
- a_r = (np.maximum(0.0, boxes[rest, 2] - boxes[rest, 0]) *
194
- np.maximum(0.0, boxes[rest, 3] - boxes[rest, 1]))
195
- iou = inter / (a_i + a_r - inter + 1e-7)
196
- order = rest[iou <= iou_thresh]
197
- return np.array(keep, dtype=np.intp)
198
-
199
- def _per_class_hard_nms(self, boxes: np.ndarray, scores: np.ndarray,
200
- cls_ids: np.ndarray) -> np.ndarray:
201
- """Per-class NMS using per-class IoU thresholds."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
202
  if len(boxes) == 0:
203
  return np.array([], dtype=np.intp)
204
- all_keep = []
205
  for c in np.unique(cls_ids):
206
  mask = cls_ids == c
207
  indices = np.where(mask)[0]
208
- cls_iou = float(self._iou_thres_array[c]) # per-class IoU threshold
209
- keep = self._hard_nms(boxes[mask], scores[mask], cls_iou)
210
  all_keep.extend(indices[keep].tolist())
211
  all_keep.sort()
212
  return np.array(all_keep, dtype=np.intp)
213
 
214
- def _cross_class_dedup_op(self, boxes: np.ndarray, scores: np.ndarray,
215
- cls_ids: np.ndarray, iou_thresh: float
216
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
217
- n = len(boxes)
218
- if n <= 1:
219
- return boxes, scores, cls_ids
220
- boxes = np.asarray(boxes, dtype=np.float32)
221
- scores = np.asarray(scores, dtype=np.float32)
222
- cls_ids = np.asarray(cls_ids, dtype=np.int32)
223
- areas = (np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) *
224
- np.maximum(0.0, boxes[:, 3] - boxes[:, 1]))
225
- margins = scores - self._conf_thres_array[cls_ids]
226
- order = np.lexsort((-areas, -margins))
227
- suppressed = np.zeros(n, dtype=bool)
228
- keep = []
229
- for i in order:
230
- if suppressed[i]:
231
- continue
232
- keep.append(int(i))
233
- bi = boxes[i]
234
- xx1 = np.maximum(bi[0], boxes[:, 0])
235
- yy1 = np.maximum(bi[1], boxes[:, 1])
236
- xx2 = np.minimum(bi[2], boxes[:, 2])
237
- yy2 = np.minimum(bi[3], boxes[:, 3])
238
- inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
239
- a_i = max(1e-7, float((bi[2] - bi[0]) * (bi[3] - bi[1])))
240
- iou = inter / (a_i + areas - inter + 1e-7)
241
- dup = iou > iou_thresh
242
- dup[i] = False
243
- suppressed |= dup
244
- keep_idx = np.array(keep, dtype=np.intp)
245
- return boxes[keep_idx], scores[keep_idx], cls_ids[keep_idx]
246
-
247
- def _filter_sane_boxes(self, boxes: np.ndarray, scores: np.ndarray,
248
- cls_ids: np.ndarray, orig_size: tuple[int, int]
249
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
250
- """Filter by per-class min area, max area ratio, and aspect ratio."""
251
  if len(boxes) == 0:
252
  return boxes, scores, cls_ids
253
-
254
  orig_w, orig_h = orig_size
255
  image_area = float(orig_w * orig_h)
256
- bw = np.maximum(0.0, boxes[:, 2] - boxes[:, 0])
257
- bh = np.maximum(0.0, boxes[:, 3] - boxes[:, 1])
258
- area = bw * bh
259
-
260
- ar = np.where(
261
- (bw > 0) & (bh > 0),
262
- np.maximum(bw / np.maximum(bh, 1e-6), bh / np.maximum(bw, 1e-6)),
263
- np.inf,
264
- )
265
-
266
- # Per-class minimum area
267
- class_min_area = self._min_box_area_array[cls_ids]
268
-
269
- keep = (
270
- (area >= class_min_area) &
271
- (area <= 0.95 * image_area)
272
- )
273
- return boxes[keep], scores[keep], cls_ids[keep]
274
-
275
- def _max_score_per_cluster(self, post_boxes: np.ndarray,
276
- post_cls: np.ndarray,
277
- full_boxes: np.ndarray,
278
- full_scores: np.ndarray,
279
- full_cls: np.ndarray,
280
- iou_thresh: float) -> np.ndarray:
281
- """For each kept box, set confidence to max score in its SAME-CLASS cluster."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
282
  n = len(post_boxes)
283
  if n == 0:
284
  return np.empty(0, dtype=np.float32)
@@ -298,16 +495,16 @@ class Miner:
298
  out[i] = float(np.max(full_scores[cluster])) if np.any(cluster) else 0.0
299
  return out
300
 
301
- def _conf_filter_mask(self, scores: np.ndarray,
302
- cls_ids: np.ndarray, extra_left: int) -> np.ndarray:
303
- """Per-class threshold with rescue bonus for missed classes."""
 
 
 
 
304
  if len(scores) == 0:
305
  return np.zeros(0, dtype=bool)
306
- thr = 0
307
- if extra_left > 0:
308
- thr = self._extra_conf_thres_array[cls_ids]
309
- else:
310
- thr = self._conf_thres_array[cls_ids]
311
  keep = scores >= thr
312
  for c in np.unique(cls_ids):
313
  b = float(self._bonus_array[c])
@@ -322,82 +519,60 @@ class Miner:
322
  keep[top] = True
323
  return keep
324
 
325
- def _suppress_overlapping_same_class(
326
  self,
327
  boxes: np.ndarray,
328
  scores: np.ndarray,
329
  cls_ids: np.ndarray,
330
- threshold: float,
331
  ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
332
- """
333
- Drop a same-class box that is (almost) entirely *contained* inside a larger
334
- same-class box — a duplicate detection of one object.
335
-
336
- Containment is intersection / area_of_SMALLER_box (IoMin), NOT IoU.
337
- A small box nested in a large one has tiny IoU, so plain NMS never removes
338
- it; IoMin catches it.
339
 
340
- black (large) + green (fully inside black) -> same gate, drop green
341
- red (sticks out of black) -> separate gate, keep
342
-
343
- Survivor = the LARGER box, and its confidence is raised to the cluster max.
 
344
  """
345
  n = len(boxes)
346
  if n <= 1:
347
  return boxes, scores, cls_ids
348
-
349
  boxes = np.asarray(boxes, dtype=np.float32)
350
- scores = np.asarray(scores, dtype=np.float32).copy()
351
  cls_ids = np.asarray(cls_ids, dtype=np.int32)
352
-
353
  areas = (np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) *
354
- np.maximum(0.0, boxes[:, 3] - boxes[:, 1]))
355
-
356
- keep = np.ones(n, dtype=bool)
357
-
358
- # Largest first, so the survivor of a containment chain is the biggest box.
359
- order = np.argsort(-areas)
360
-
361
- for idx_a in range(n):
362
- a = order[idx_a]
363
- if not keep[a]:
364
  continue
365
- for idx_b in range(idx_a + 1, n):
366
- b = order[idx_b] # areas[b] <= areas[a]
367
- if not keep[b]:
368
- continue
369
- if cls_ids[a] != cls_ids[b]:
370
- continue
371
-
372
- x1 = max(boxes[a, 0], boxes[b, 0])
373
- y1 = max(boxes[a, 1], boxes[b, 1])
374
- x2 = min(boxes[a, 2], boxes[b, 2])
375
- y2 = min(boxes[a, 3], boxes[b, 3])
376
- if x2 <= x1 or y2 <= y1:
377
- continue
378
- inter = (x2 - x1) * (y2 - y1)
379
-
380
- # How much of the SMALLER box (b) lies inside the larger (a):
381
- containment_b = inter / max(areas[b], 1e-9)
382
-
383
- if containment_b >= threshold: # b is nested -> it's a duplicate
384
- scores[a] = max(scores[a], scores[b]) # keep the higher score
385
- keep[b] = False # drop the smaller (green)
386
-
387
- keep_idx = np.where(keep)[0]
388
  return boxes[keep_idx], scores[keep_idx], cls_ids[keep_idx]
389
 
390
- def _per_view_pipeline(self, boxes: np.ndarray, scores: np.ndarray,
391
- cls_ids: np.ndarray, orig_size: tuple[int, int]
392
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
393
- """Sanity filter + per-class NMS + cross-class dedup."""
394
- boxes, scores, cls_ids = self._filter_sane_boxes(
395
- boxes, scores, cls_ids, orig_size
396
- )
397
- if len(boxes) == 0:
398
- return boxes, scores, cls_ids
399
  if len(boxes) > 1:
400
- keep = self._per_class_hard_nms(boxes, scores, cls_ids)
401
  boxes, scores, cls_ids = boxes[keep], scores[keep], cls_ids[keep]
402
  if len(scores) > self.max_det:
403
  top = np.argsort(-scores)[: self.max_det]
@@ -408,36 +583,32 @@ class Miner:
408
  )
409
  return boxes, scores, cls_ids
410
 
411
- # ─── v26-specific decoding ────────────────────────────────────
412
-
413
- def _decode_v26_output(self, preds: np.ndarray, ratio: float,
414
- pad: tuple[float, float],
415
- orig_size: tuple[int, int],
416
- extra: tuple[int, int] = (0, 0)) -> list[BoundingBox]: # NEW arg
417
- """Decode YOLOv26 output (shape [1, 300, 6] or [300, 6])."""
 
 
 
 
 
 
418
  if preds.ndim == 3 and preds.shape[0] == 1:
419
  preds = preds[0]
420
 
421
  if preds.ndim != 2 or preds.shape[1] < 6:
422
- print(f"Warning: Unexpected v26 output shape: {preds.shape}")
423
- return []
424
 
425
  boxes = preds[:, :4].astype(np.float32)
426
  scores = preds[:, 4].astype(np.float32)
427
  cls_ids = preds[:, 5].astype(np.int32)
 
428
 
429
- n_cls = len(self.class_names)
430
- valid = (cls_ids >= 0) & (cls_ids < n_cls)
431
- boxes = boxes[valid]
432
- scores = scores[valid]
433
- cls_ids = cls_ids[valid]
434
-
435
- if len(boxes) == 0:
436
- return []
437
-
438
- extra_left, _extra_right = extra
439
-
440
- keep = self._conf_filter_mask(scores, cls_ids, extra_left)
441
  boxes = boxes[keep]
442
  scores = scores[keep]
443
  cls_ids = cls_ids[keep]
@@ -445,169 +616,262 @@ class Miner:
445
  if len(boxes) == 0:
446
  return []
447
 
448
- # 1) undo letterbox -> coords in the PADDED (widened) image
449
  pad_w, pad_h = pad
 
 
 
450
  boxes[:, [0, 2]] -= pad_w
451
  boxes[:, [1, 3]] -= pad_h
452
  boxes /= ratio
 
453
 
454
- # 2) NEW: undo the left/right pre-padding -> coords in the ORIGINAL image.
455
- # Only the LEFT pad shifts x; right pad adds width but no offset.
456
-
457
- if extra_left:
458
- boxes[:, [0, 2]] -= extra_left
459
-
460
- # 2b) NEW: drop boxes that fall in the black padding region.
461
- # A real detection must have its CENTER inside the original image
462
- # width [0, orig_w]; boxes centered in the black bars are spurious.
463
- if extra_left or _extra_right:
464
- orig_w, orig_h = orig_size
465
- cx = (boxes[:, 0] + boxes[:, 2]) * 0.5
466
- inside = (cx >= 0) & (cx <= orig_w)
467
- boxes = boxes[inside]
468
- scores = scores[inside]
469
- cls_ids = cls_ids[inside]
470
- if len(boxes) == 0:
471
- return []
472
-
473
- # 3) clip to ORIGINAL image bounds (orig_size is the true original size)
474
- boxes = self._clip_boxes(boxes, orig_size)
475
-
476
- boxes, scores, cls_ids = self._per_view_pipeline(
477
  boxes, scores, cls_ids, orig_size
478
  )
 
 
479
 
480
- return self._build_results(boxes, scores, cls_ids, orig_size)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
481
 
482
- @staticmethod
483
- def _build_results(boxes: np.ndarray, scores: np.ndarray,
484
- cls_ids: np.ndarray,
485
- orig_size: tuple[int, int]) -> list[BoundingBox]:
486
- results = []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
487
  orig_w, orig_h = orig_size
 
 
 
 
 
 
 
 
 
 
 
 
 
 
488
  for box, conf, cls_id in zip(boxes, scores, cls_ids):
489
  x1, y1, x2, y2 = box.tolist()
 
490
  if x2 <= x1 or y2 <= y1:
491
  continue
 
492
  results.append(
493
  BoundingBox(
494
- x1=max(0, min(orig_w, int(math.floor(x1)))),
495
- y1=max(0, min(orig_h, int(math.floor(y1)))),
496
- x2=max(0, min(orig_w, int(math.ceil(x2)))),
497
- y2=max(0, min(orig_h, int(math.ceil(y2)))),
498
  cls_id=int(cls_id),
499
- conf=float(max(0.0, min(1.0, conf))),
500
  )
501
  )
 
502
  return results
503
 
504
- # ─── Single-view inference ────────────────────────────────────
505
-
506
- def _predict_single(self, image_bgr: np.ndarray,
507
- allow_pad: bool = True) -> list[BoundingBox]:
508
- if image_bgr is None or not isinstance(image_bgr, np.ndarray):
509
- raise ValueError("Invalid image input")
510
- if image_bgr.dtype != np.uint8:
511
- image_bgr = image_bgr.astype(np.uint8)
512
-
513
- inp, meta = self._preprocess(image_bgr, allow_pad=allow_pad)
514
- outputs = self.session.run(None, {self.input_name: inp})
515
-
516
- ratio = float(meta["ratio"])
517
- pad = (float(meta["pad_w"]), float(meta["pad_h"]))
518
- orig_size = (int(meta["orig_w"]), int(meta["orig_h"]))
519
- extra = (int(meta["extra_left"]), int(meta["extra_right"]))
520
-
521
- return self._decode_v26_output(outputs[0], ratio, pad, orig_size, extra)
522
-
523
- # ─── TTA inference ────────────────────────────────────────────
524
-
525
- def _infer_single(self, image_bgr: ndarray) -> list[BoundingBox]:
526
- """3-view TTA: original (no pad) + flip (no pad) + original (L/R padded).
527
- All three views return ORIGINAL-image coords, then pooled."""
528
- orig_h, orig_w = image_bgr.shape[:2]
529
-
530
- # View 1: original, NO left/right padding
531
- boxes_orig = self._predict_single(image_bgr, allow_pad=True)
532
-
533
- # View 2: horizontal flip, NO left/right padding
534
- flipped = cv2.flip(image_bgr, 1)
535
- boxes_flip = self._predict_single(flipped, allow_pad=True)
536
- w = image_bgr.shape[1]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
537
  boxes_flip = [
538
- BoundingBox(x1=w - b.x2, y1=b.y1, x2=w - b.x1, y2=b.y2,
539
- cls_id=b.cls_id, conf=b.conf)
 
 
540
  for b in boxes_flip
541
  ]
542
-
543
- # View 3: original, WITH left/right padding (only meaningful if square)
544
- # boxes_pad = self._predict_single(image_bgr, allow_pad=True)
545
- # NOZZLE_CLS = self.class_names.index('nozzle') # == 2
546
- # boxes_pad = [b for b in boxes_pad if b.cls_id != NOZZLE_CLS]
547
-
548
- # # View 4: flip, WITH left/right padding (only meaningful if square)
549
- # flipped = cv2.flip(image_bgr, 1)
550
- # boxes_pad_flip = self._predict_single(flipped, allow_pad=True)
551
- # boxes_pad_flip = [
552
- # BoundingBox(x1=w - b.x2, y1=b.y1, x2=w - b.x1, y2=b.y2,
553
- # cls_id=b.cls_id, conf=b.conf)
554
- # for b in boxes_pad_flip
555
- # ]
556
- # NOZZLE_CLS = self.class_names.index('nozzle') # == 2
557
- # boxes_pad_flip = [b for b in boxes_pad_flip if b.cls_id != NOZZLE_CLS]
558
-
559
- all_boxes = boxes_orig + boxes_flip# + boxes_pad# + boxes_pad_flip
560
- if not all_boxes:
561
  return []
562
 
563
- coords = np.array([[b.x1, b.y1, b.x2, b.y2] for b in all_boxes], dtype=np.float32)
 
 
564
  scores = np.array([b.conf for b in all_boxes], dtype=np.float32)
565
  cls_ids = np.array([b.cls_id for b in all_boxes], dtype=np.int32)
566
 
567
- # Per-class NMS (uses per-class IoU thresholds)
568
- hard_keep = self._per_class_hard_nms(coords, scores, cls_ids)
569
  if len(hard_keep) == 0:
570
  return []
571
-
572
  if len(hard_keep) > self.max_det:
573
  top = np.argsort(-scores[hard_keep])[: self.max_det]
574
  hard_keep = hard_keep[top]
575
-
576
- # For confidence boost, use average IoU threshold (or could use median)
577
- avg_iou = float(np.mean(self._iou_thres_array))
578
  boosted = self._max_score_per_cluster(
579
  coords[hard_keep], cls_ids[hard_keep],
580
- coords, scores, cls_ids, avg_iou,
581
  )
582
 
583
  kept_coords = coords[hard_keep]
584
  kept_cls = cls_ids[hard_keep]
585
-
586
- # Cross-class dedup
587
  if len(kept_coords) > 1:
588
  kept_coords, boosted, kept_cls = self._cross_class_dedup_op(
589
  kept_coords, boosted, kept_cls, self.cross_iou_thresh
590
  )
591
 
592
- out_boxes = []
593
- for j in range(len(kept_coords)):
594
- x1, y1, x2, y2 = kept_coords[j].tolist()
595
- if x2 <= x1 or y2 <= y1:
596
- continue
597
- out_boxes.append(
598
- BoundingBox(
599
- x1=max(0, min(orig_w, int(math.floor(x1)))),
600
- y1=max(0, min(orig_h, int(math.floor(y1)))),
601
- x2=max(0, min(orig_w, int(math.ceil(x2)))),
602
- y2=max(0, min(orig_h, int(math.ceil(y2)))),
603
- cls_id=int(kept_cls[j]),
604
- conf=float(max(0.0, min(1.0, boosted[j]))),
605
- )
606
  )
607
- return out_boxes
 
608
 
609
- # ─── Public API ───────────────────────────────────────────────
610
-
611
  def predict_batch(
612
  self,
613
  batch_images: list[ndarray],
@@ -615,18 +879,23 @@ class Miner:
615
  n_keypoints: int,
616
  ) -> list[TVFrameResult]:
617
  results: list[TVFrameResult] = []
618
- for idx, image in enumerate(batch_images):
 
619
  try:
620
- boxes = self._infer_single(image)
 
 
 
621
  except Exception as e:
622
- print(f"Inference failed for frame {offset + idx}: {e}")
623
  boxes = []
624
- keypoints = [(0, 0) for _ in range(max(0, int(n_keypoints)))]
625
  results.append(
626
  TVFrameResult(
627
- frame_id=offset + idx,
628
  boxes=boxes,
629
- keypoints=keypoints,
630
  )
631
  )
 
632
  return results
 
24
 
25
 
26
  class Miner:
27
+ def __init__(self,
28
+ path_hf_repo: Path
29
+ ) -> None:
30
+ model_path = self._resolve_model_path(path_hf_repo)
31
+ # car-wash element classes — cls_id order MUST match element `objects`
32
+ # (0=broom, 1=drainage gate, 2=nozzle, 3=track). This is the canonical
33
+ # order every downstream consumer (validator, BoundingBox.cls_id) sees.
34
+ self.class_names = ["broom", "drainage gate", "nozzle", "track"]
35
+ # FALLBACK model-emit order: the authoritative order is read from the
36
+ # ONNX `names` metadata after the session is created (embedded by
37
+ # Ultralytics at export, ships inside weights.onnx), so a retrained
38
+ # model with a different class order is remapped correctly without
39
+ # code changes. This list is used only when metadata is missing.
40
+ self._model_class_order = ["broom", "drainage gate", "nozzle", "track"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
41
  print("ORT version:", ort.__version__)
42
+
43
  try:
44
  ort.preload_dlls()
45
+ print("✅ onnxruntime.preload_dlls() success")
46
  except Exception as e:
47
+ print(f"⚠️ preload_dlls failed: {e}")
48
+
49
  print("ORT available providers BEFORE session:", ort.get_available_providers())
50
+
51
  sess_options = ort.SessionOptions()
52
  sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
53
+ sess_options.intra_op_num_threads = 2
54
+ sess_options.inter_op_num_threads = 1
55
+ sess_options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
56
+
57
+ try:
58
+ self.session = ort.InferenceSession(
59
+ str(model_path),
60
+ sess_options=sess_options,
61
+ providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
62
+ )
63
+ print("✅ Created ORT session with preferred CUDA provider list")
64
+ except Exception as e:
65
+ print(f"⚠️ CUDA session creation failed, falling back to CPU: {e}")
66
+ self.session = ort.InferenceSession(
67
+ str(model_path),
68
+ sess_options=sess_options,
69
+ providers=["CPUExecutionProvider"],
70
+ )
71
+
72
  print("ORT session providers:", self.session.get_providers())
73
+
74
+ # Build cls_remap: for each model-emit index i,
75
+ # cls_remap[i] = self.class_names.index(model_class_order[i])
76
+ # The model-side order comes from the ONNX metadata when available,
77
+ # else falls back to the static _model_class_order.
78
+ model_class_order = self._read_model_class_order()
79
+ if model_class_order is None:
80
+ model_class_order = list(self._model_class_order)
81
+ print(f"cls order: no usable ONNX metadata, FALLBACK {model_class_order}")
82
+ else:
83
+ print(f"cls order: from ONNX metadata {model_class_order}")
84
+ self.cls_remap = np.array(
85
+ [self.class_names.index(n) for n in model_class_order], dtype=np.int32
86
+ )
87
+
88
+ for inp in self.session.get_inputs():
89
+ print("INPUT:", inp.name, inp.shape, inp.type)
90
+
91
+ for out in self.session.get_outputs():
92
+ print("OUTPUT:", out.name, out.shape, out.type)
93
+
94
  self.input_name = self.session.get_inputs()[0].name
95
  self.output_names = [output.name for output in self.session.get_outputs()]
96
+ self.input_shape = self.session.get_inputs()[0].shape
97
+
98
+ # Match the ONNX input dtype (this export is FP16 -> needs float16 input).
99
+ input_type = self.session.get_inputs()[0].type
100
+ self.np_dtype = np.float16 if "float16" in input_type else np.float32
101
+ print(f"✅ ONNX input dtype: {input_type} -> numpy {self.np_dtype}")
102
+
103
+ # ONNX is fixed-size 1408x1408 (v1 export); read actual shape to be safe.
104
+ self.input_height = self._safe_dim(self.input_shape[2], default=1280)
105
+ self.input_width = self._safe_dim(self.input_shape[3], default=1280)
106
+
107
+ # Tuned for validator scoring (pillars: 0.6*map50 + 0.4*false_positive).
108
+ # All values below are the measured optimum of a full grid sweep on
109
+ # the validator-style val split (tune_miner.py, 241 1024x1024 crops,
110
+ # composite 0.8002 -> 0.8103) -- re-run the sweep after any retrain.
111
+ self.iou_thres = 0.5 # Per-class NMS IoU; lower = stricter dedup
112
+ self.cross_iou_thresh = 0.9 # Cross-class dedup IoU (suppress same physical object firing multiple classes)
113
+ self.max_det = 200
114
+ # TTA = a 2nd (flipped) forward pass. Doubles latency; off for the
115
+ # CPU latency gate. Re-enable only if the latency budget allows.
116
+ self.use_tta = True
117
+
118
+ # conf thresholds: broom=0.38 drainage gate=0.45 nozzle=0.30 track=0.60
119
+ # Per-class confidence thresholds.
120
+ # Indexed by class_names order: [broom, drainage gate, nozzle, track].
121
+ # broom/nozzle sit low: under the validator metric the mAP gained
122
+ # from the extra recall outweighs the FP-pillar cost (the previous
123
+ # 0.5/0.5 silently discarded many valid detections); track is the
124
+ # one class where false fires are common enough to need 0.38.
125
+ self._conf_thres_array = np.array(
126
+ [0.28, 0.38, 0.60, 0.45], dtype=np.float32
127
+ )
128
+ # Per-class rescue bonus: when a class has ZERO boxes passing the
129
+ # threshold in a frame, its top-1 candidate is admitted when its score
130
+ # is at least (per-class threshold - per-class bonus).
131
+ # DISABLED (all zeros): the sweep showed rescue admits more false
132
+ # positives than true positives under the validator's FP pillar.
133
+ self._bonus_array = np.array(
134
+ [0.05, 0.1, 0.25, 0.2], dtype=np.float32
135
+ )
136
+
137
+ # Box sanity filter — kept loose: car-wash `nozzle` boxes are tiny
138
+ # (GT median ~290 px², smallest ~32 px²). Fire's 14x14/min_side 8
139
+ # would delete valid nozzles, so thresholds are dropped here.
140
+ self.min_box_area = 4 * 4 # 16 px²
141
+ self.min_side = 3
142
+ self.max_aspect_ratio = 12.0
143
+
144
+ print(f"✅ ONNX model loaded from: {model_path}")
145
+ print(f"✅ ONNX providers: {self.session.get_providers()}")
146
+ print(f"✅ ONNX input: name={self.input_name}, shape={self.input_shape}")
147
 
148
  def __repr__(self) -> str:
149
+ return (
150
+ f"ONNXRuntime(session={type(self.session).__name__}, "
151
+ f"providers={self.session.get_providers()})"
152
+ )
153
 
154
  @staticmethod
155
  def _safe_dim(value, default: int) -> int:
156
  return value if isinstance(value, int) and value > 0 else default
157
 
158
+ @staticmethod
159
+ def _resolve_model_path(repo: Path) -> Path:
160
+ """Locate the ONNX model in the repo dir.
161
+
162
+ Prefers weights.onnx (FP16/FP32 export), then weights_int8.onnx (the
163
+ training script's INT8-quantized export -- works as-is: quantization
164
+ preserves the Ultralytics metadata and QDQ models take regular fp32
165
+ input), then any other .onnx file. INT8 is the fallback when the FP16
166
+ export exceeds the 30 MB deployment limit (e.g. yolo26m).
167
+ """
168
+ for name in ("weights.onnx", "weights_int8.onnx"):
169
+ p = repo / name
170
+ if p.exists():
171
+ if name != "weights.onnx":
172
+ print(f"model: weights.onnx not found, using {name}")
173
+ return p
174
+ candidates = sorted(repo.glob("*.onnx"))
175
+ if candidates:
176
+ print(f"model: using {candidates[0].name}")
177
+ return candidates[0]
178
+ return repo / "weights.onnx" # let session creation raise the error
179
+
180
+ def _read_model_class_order(self) -> list[str] | None:
181
+ """Read the model's class order from Ultralytics ONNX metadata.
182
+
183
+ Returns the class names ordered by model-emit index, or None when
184
+ metadata is missing/unparsable or doesn't match `class_names` as a
185
+ set (in which case the static _model_class_order fallback is used).
186
+ """
187
+ try:
188
+ import ast
189
+
190
+ meta = self.session.get_modelmeta().custom_metadata_map
191
+ names = ast.literal_eval(meta["names"]) # e.g. {0: 'broom', ...}
192
+ if isinstance(names, dict):
193
+ order = [str(names[i]) for i in sorted(names)]
194
+ else:
195
+ order = [str(n) for n in names]
196
+ except Exception as e:
197
+ print(f"cls order: could not read ONNX names metadata ({e})")
198
+ return None
199
+ if sorted(order) != sorted(self.class_names):
200
+ print(
201
+ f"cls order: ONNX names {order} do not match expected classes "
202
+ f"{self.class_names}; ignoring metadata"
203
+ )
204
+ return None
205
+ return order
206
+
207
  def _letterbox(
208
+ self,
209
+ image: ndarray,
210
+ new_shape: tuple[int, int],
211
+ color=(114, 114, 114),
212
+ ) -> tuple[ndarray, float, tuple[float, float]]:
213
+ """
214
+ Resize with unchanged aspect ratio and pad to target shape.
215
+ Returns:
216
+ padded_image,
217
+ ratio,
218
+ (pad_w, pad_h) # half-padding
219
+ """
220
+ h, w = image.shape[:2]
221
+ new_w, new_h = new_shape
222
+
223
+ ratio = min(new_w / w, new_h / h)
224
+ resized_w = int(round(w * ratio))
225
+ resized_h = int(round(h * ratio))
226
+
227
+ if (resized_w, resized_h) != (w, h):
228
+ interp = cv2.INTER_CUBIC if ratio > 1.0 else cv2.INTER_LINEAR
229
+ image = cv2.resize(image, (resized_w, resized_h), interpolation=interp)
230
+
231
+ dw = new_w - resized_w
232
+ dh = new_h - resized_h
233
+ dw /= 2.0
234
+ dh /= 2.0
235
+
236
+ left = int(round(dw - 0.1))
237
+ right = int(round(dw + 0.1))
238
+ top = int(round(dh - 0.1))
239
+ bottom = int(round(dh + 0.1))
240
+
241
+ padded = cv2.copyMakeBorder(
242
+ image,
243
+ top,
244
+ bottom,
245
+ left,
246
+ right,
247
+ borderType=cv2.BORDER_CONSTANT,
248
+ value=color,
249
+ )
250
+ return padded, ratio, (dw, dh)
251
+
252
+ def _preprocess(
253
+ self, image: ndarray
254
+ ) -> tuple[np.ndarray, float, tuple[float, float], tuple[int, int]]:
255
+ """
256
+ Preprocess for fixed-size ONNX export:
257
+ - enhance image quality (CLAHE, denoise, sharpen)
258
+ - letterbox to model input size
259
+ - BGR -> RGB
260
+ - normalize to [0,1]
261
+ - HWC -> NCHW float32
262
+ """
263
  orig_h, orig_w = image.shape[:2]
264
+
265
+ img, ratio, pad = self._letterbox(
266
+ image, (self.input_width, self.input_height)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
267
  )
268
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
269
+ img = (img.astype(np.float32) / 255.0)
270
+ img = np.transpose(img, (2, 0, 1))[None, ...]
271
+ img = np.ascontiguousarray(img, dtype=self.np_dtype)
272
+
273
+ return img, ratio, pad, (orig_w, orig_h)
274
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
275
  @staticmethod
276
  def _clip_boxes(boxes: np.ndarray, image_size: tuple[int, int]) -> np.ndarray:
277
  w, h = image_size
 
282
  return boxes
283
 
284
  @staticmethod
285
+ def _xywh_to_xyxy(boxes: np.ndarray) -> np.ndarray:
286
+ out = np.empty_like(boxes)
287
+ out[:, 0] = boxes[:, 0] - boxes[:, 2] / 2.0
288
+ out[:, 1] = boxes[:, 1] - boxes[:, 3] / 2.0
289
+ out[:, 2] = boxes[:, 0] + boxes[:, 2] / 2.0
290
+ out[:, 3] = boxes[:, 1] + boxes[:, 3] / 2.0
291
+ return out
292
+
293
+ def _soft_nms(
294
+ self,
295
+ boxes: np.ndarray,
296
+ scores: np.ndarray,
297
+ sigma: float = 0.5,
298
+ score_thresh: float = 0.01,
299
+ ) -> tuple[np.ndarray, np.ndarray]:
300
+ """
301
+ Soft-NMS: Gaussian decay of overlapping scores instead of hard removal.
302
+ Returns (kept_original_indices, updated_scores).
303
+ """
304
+ N = len(boxes)
305
+ if N == 0:
306
+ return np.array([], dtype=np.intp), np.array([], dtype=np.float32)
307
+
308
+ boxes = boxes.astype(np.float32, copy=True)
309
+ scores = scores.astype(np.float32, copy=True)
310
+ order = np.arange(N)
311
+
312
+ for i in range(N):
313
+ max_pos = i + int(np.argmax(scores[i:]))
314
+ boxes[[i, max_pos]] = boxes[[max_pos, i]]
315
+ scores[[i, max_pos]] = scores[[max_pos, i]]
316
+ order[[i, max_pos]] = order[[max_pos, i]]
317
+
318
+ if i + 1 >= N:
319
  break
320
+
321
+ xx1 = np.maximum(boxes[i, 0], boxes[i + 1:, 0])
322
+ yy1 = np.maximum(boxes[i, 1], boxes[i + 1:, 1])
323
+ xx2 = np.minimum(boxes[i, 2], boxes[i + 1:, 2])
324
+ yy2 = np.minimum(boxes[i, 3], boxes[i + 1:, 3])
325
  inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
326
+
327
+ area_i = max(0.0, float(
328
+ (boxes[i, 2] - boxes[i, 0]) * (boxes[i, 3] - boxes[i, 1])
329
+ ))
330
+ areas_j = (
331
+ np.maximum(0.0, boxes[i + 1:, 2] - boxes[i + 1:, 0])
332
+ * np.maximum(0.0, boxes[i + 1:, 3] - boxes[i + 1:, 1])
333
+ )
334
+ iou = inter / (area_i + areas_j - inter + 1e-7)
335
+ scores[i + 1:] *= np.exp(-(iou ** 2) / sigma)
336
+
337
+ mask = scores > score_thresh
338
+ return order[mask], scores[mask]
339
+
340
+ @staticmethod
341
+ def _hard_nms(
342
+ boxes: np.ndarray,
343
+ scores: np.ndarray,
344
+ iou_thresh: float,
345
+ ) -> np.ndarray:
346
+ """
347
+ Standard NMS: keep one box per overlapping cluster (the one with highest score).
348
+ Returns indices of kept boxes (into the boxes/scores arrays).
349
+ """
350
+ N = len(boxes)
351
+ if N == 0:
352
+ return np.array([], dtype=np.intp)
353
+ boxes = np.asarray(boxes, dtype=np.float32)
354
+ scores = np.asarray(scores, dtype=np.float32)
355
+ order = np.argsort(scores)[::-1]
356
+ keep: list[int] = []
357
+ suppressed = np.zeros(N, dtype=bool)
358
+ for i in range(N):
359
+ idx = order[i]
360
+ if suppressed[idx]:
361
+ continue
362
+ keep.append(idx)
363
+ bi = boxes[idx]
364
+ for k in range(i + 1, N):
365
+ jdx = order[k]
366
+ if suppressed[jdx]:
367
+ continue
368
+ bj = boxes[jdx]
369
+ xx1 = max(bi[0], bj[0])
370
+ yy1 = max(bi[1], bj[1])
371
+ xx2 = min(bi[2], bj[2])
372
+ yy2 = min(bi[3], bj[3])
373
+ inter = max(0.0, xx2 - xx1) * max(0.0, yy2 - yy1)
374
+ area_i = (bi[2] - bi[0]) * (bi[3] - bi[1])
375
+ area_j = (bj[2] - bj[0]) * (bj[3] - bj[1])
376
+ iou = inter / (area_i + area_j - inter + 1e-7)
377
+ if iou > iou_thresh:
378
+ suppressed[jdx] = True
379
+ return np.array(keep)
380
+
381
+ def _per_class_hard_nms(
382
+ self,
383
+ boxes: np.ndarray,
384
+ scores: np.ndarray,
385
+ cls_ids: np.ndarray,
386
+ iou_thresh: float,
387
+ ) -> np.ndarray:
388
+ """Hard NMS applied independently per class."""
389
  if len(boxes) == 0:
390
  return np.array([], dtype=np.intp)
391
+ all_keep: list[int] = []
392
  for c in np.unique(cls_ids):
393
  mask = cls_ids == c
394
  indices = np.where(mask)[0]
395
+ keep = self._hard_nms(boxes[mask], scores[mask], iou_thresh)
 
396
  all_keep.extend(indices[keep].tolist())
397
  all_keep.sort()
398
  return np.array(all_keep, dtype=np.intp)
399
 
400
+ def _per_class_soft_nms(
401
+ self,
402
+ boxes: np.ndarray,
403
+ scores: np.ndarray,
404
+ cls_ids: np.ndarray,
405
+ sigma: float = 0.5,
406
+ score_thresh: float = 0.01,
407
+ ) -> tuple[np.ndarray, np.ndarray]:
408
+ """Soft NMS applied independently per class."""
409
+ if len(boxes) == 0:
410
+ return np.array([], dtype=np.intp), np.array([], dtype=np.float32)
411
+ all_keep: list[int] = []
412
+ all_scores: list[float] = []
413
+ for c in np.unique(cls_ids):
414
+ mask = cls_ids == c
415
+ indices = np.where(mask)[0]
416
+ keep, updated = self._soft_nms(boxes[mask], scores[mask], sigma, score_thresh)
417
+ for k, s in zip(keep, updated):
418
+ all_keep.append(int(indices[k]))
419
+ all_scores.append(float(s))
420
+ if not all_keep:
421
+ return np.array([], dtype=np.intp), np.array([], dtype=np.float32)
422
+ return np.array(all_keep, dtype=np.intp), np.array(all_scores, dtype=np.float32)
423
+
424
+ def _filter_sane_boxes(
425
+ self,
426
+ boxes: np.ndarray,
427
+ scores: np.ndarray,
428
+ cls_ids: np.ndarray,
429
+ orig_size: tuple[int, int],
430
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
431
+ """Filter out tiny, degenerate, or implausible boxes (common FP)."""
 
 
 
 
 
432
  if len(boxes) == 0:
433
  return boxes, scores, cls_ids
 
434
  orig_w, orig_h = orig_size
435
  image_area = float(orig_w * orig_h)
436
+ keep = []
437
+ for i, box in enumerate(boxes):
438
+ x1, y1, x2, y2 = box.tolist()
439
+ bw = x2 - x1
440
+ bh = y2 - y1
441
+ if bw <= 0 or bh <= 0:
442
+ continue
443
+ if bw < self.min_side or bh < self.min_side:
444
+ continue
445
+ area = bw * bh
446
+ if area < self.min_box_area:
447
+ continue
448
+ if area > 0.95 * image_area:
449
+ continue
450
+ ar = max(bw / max(bh, 1e-6), bh / max(bw, 1e-6))
451
+ if ar > self.max_aspect_ratio:
452
+ continue
453
+ keep.append(i)
454
+ if not keep:
455
+ return (
456
+ np.empty((0, 4), dtype=np.float32),
457
+ np.empty((0,), dtype=np.float32),
458
+ np.empty((0,), dtype=np.int32),
459
+ )
460
+ k = np.array(keep, dtype=np.intp)
461
+ return boxes[k], scores[k], cls_ids[k]
462
+
463
+ @staticmethod
464
+ def _max_score_per_cluster(
465
+ post_boxes: np.ndarray,
466
+ post_cls: np.ndarray,
467
+ full_boxes: np.ndarray,
468
+ full_scores: np.ndarray,
469
+ full_cls: np.ndarray,
470
+ iou_thresh: float,
471
+ ) -> np.ndarray:
472
+ """For each kept (post-NMS) box, return the max score over the FULL
473
+ candidate set among SAME-CLASS boxes with IoU >= iou_thresh.
474
+
475
+ The previous version omitted the same-class constraint, which let a
476
+ confident broom raise the score of a coincident nozzle (or vice
477
+ versa) under TTA. That's a silent FP booster and is fixed here.
478
+ """
479
  n = len(post_boxes)
480
  if n == 0:
481
  return np.empty(0, dtype=np.float32)
 
495
  out[i] = float(np.max(full_scores[cluster])) if np.any(cluster) else 0.0
496
  return out
497
 
498
+ def _conf_filter_mask(
499
+ self, scores: np.ndarray, cls_ids: np.ndarray
500
+ ) -> np.ndarray:
501
+ """Boolean keep-mask: score >= per-class threshold, with a per-class
502
+ rescue -- if a class has zero boxes passing, admit its top-1 candidate
503
+ when its score >= (per-class threshold - per-class bonus).
504
+ """
505
  if len(scores) == 0:
506
  return np.zeros(0, dtype=bool)
507
+ thr = self._conf_thres_array[cls_ids]
 
 
 
 
508
  keep = scores >= thr
509
  for c in np.unique(cls_ids):
510
  b = float(self._bonus_array[c])
 
519
  keep[top] = True
520
  return keep
521
 
522
+ def _cross_class_dedup_op(
523
  self,
524
  boxes: np.ndarray,
525
  scores: np.ndarray,
526
  cls_ids: np.ndarray,
527
+ iou_thresh: float,
528
  ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
529
+ """Remove near-duplicate boxes across classes.
 
 
 
 
 
 
530
 
531
+ Order candidates by (score - per_class_threshold) margin, then by area;
532
+ keep the highest, suppress every other box with IoU > iou_thresh. For
533
+ car-wash this kills the common failure where water spray makes the
534
+ model fire both `nozzle` and `track` on the same patch, or where a
535
+ broom handle overlaps a drainage-gate detection.
536
  """
537
  n = len(boxes)
538
  if n <= 1:
539
  return boxes, scores, cls_ids
 
540
  boxes = np.asarray(boxes, dtype=np.float32)
541
+ scores = np.asarray(scores, dtype=np.float32)
542
  cls_ids = np.asarray(cls_ids, dtype=np.int32)
 
543
  areas = (np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) *
544
+ np.maximum(0.0, boxes[:, 3] - boxes[:, 1]))
545
+ margins = scores - self._conf_thres_array[cls_ids]
546
+ order = np.lexsort((-areas, -margins))
547
+ suppressed = np.zeros(n, dtype=bool)
548
+ keep: list[int] = []
549
+ for i in order:
550
+ if suppressed[i]:
 
 
 
551
  continue
552
+ keep.append(int(i))
553
+ bi = boxes[i]
554
+ xx1 = np.maximum(bi[0], boxes[:, 0])
555
+ yy1 = np.maximum(bi[1], boxes[:, 1])
556
+ xx2 = np.minimum(bi[2], boxes[:, 2])
557
+ yy2 = np.minimum(bi[3], boxes[:, 3])
558
+ inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
559
+ a_i = max(1e-7, float((bi[2] - bi[0]) * (bi[3] - bi[1])))
560
+ iou = inter / (a_i + areas - inter + 1e-7)
561
+ dup = iou > iou_thresh
562
+ dup[i] = False
563
+ suppressed |= dup
564
+ keep_idx = np.array(keep, dtype=np.intp)
 
 
 
 
 
 
 
 
 
 
565
  return boxes[keep_idx], scores[keep_idx], cls_ids[keep_idx]
566
 
567
+ def _per_view_pipeline(
568
+ self,
569
+ boxes: np.ndarray,
570
+ scores: np.ndarray,
571
+ cls_ids: np.ndarray,
572
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
573
+ """Per-view post-processing: per-class NMS -> cap -> cross-class dedup."""
 
 
574
  if len(boxes) > 1:
575
+ keep = self._per_class_hard_nms(boxes, scores, cls_ids, self.iou_thres)
576
  boxes, scores, cls_ids = boxes[keep], scores[keep], cls_ids[keep]
577
  if len(scores) > self.max_det:
578
  top = np.argsort(-scores)[: self.max_det]
 
583
  )
584
  return boxes, scores, cls_ids
585
 
586
+ def _decode_final_dets(
587
+ self,
588
+ preds: np.ndarray,
589
+ ratio: float,
590
+ pad: tuple[float, float],
591
+ orig_size: tuple[int, int],
592
+ apply_optional_dedup: bool = False,
593
+ ) -> list[BoundingBox]:
594
+ """
595
+ Primary path:
596
+ expected output rows like [x1, y1, x2, y2, conf, cls_id]
597
+ in letterboxed input coordinates.
598
+ """
599
  if preds.ndim == 3 and preds.shape[0] == 1:
600
  preds = preds[0]
601
 
602
  if preds.ndim != 2 or preds.shape[1] < 6:
603
+ raise ValueError(f"Unexpected ONNX final-det output shape: {preds.shape}")
 
604
 
605
  boxes = preds[:, :4].astype(np.float32)
606
  scores = preds[:, 4].astype(np.float32)
607
  cls_ids = preds[:, 5].astype(np.int32)
608
+ cls_ids = self.cls_remap[cls_ids]
609
 
610
+ # Per-class confidence filter with rescue (replaces scalar threshold)
611
+ keep = self._conf_filter_mask(scores, cls_ids)
 
 
 
 
 
 
 
 
 
 
612
  boxes = boxes[keep]
613
  scores = scores[keep]
614
  cls_ids = cls_ids[keep]
 
616
  if len(boxes) == 0:
617
  return []
618
 
 
619
  pad_w, pad_h = pad
620
+ orig_w, orig_h = orig_size
621
+
622
+ # reverse letterbox
623
  boxes[:, [0, 2]] -= pad_w
624
  boxes[:, [1, 3]] -= pad_h
625
  boxes /= ratio
626
+ boxes = self._clip_boxes(boxes, (orig_w, orig_h))
627
 
628
+ # Box sanity filter (reduces FP)
629
+ boxes, scores, cls_ids = self._filter_sane_boxes(
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
630
  boxes, scores, cls_ids, orig_size
631
  )
632
+ if len(boxes) == 0:
633
+ return []
634
 
635
+ if apply_optional_dedup and len(boxes) > 1:
636
+ # Soft-NMS path preserved as a tunable option; default below.
637
+ keep_idx, scores = self._per_class_soft_nms(boxes, scores, cls_ids)
638
+ boxes = boxes[keep_idx]
639
+ cls_ids = cls_ids[keep_idx]
640
+ if len(scores) > self.max_det:
641
+ top = np.argsort(-scores)[: self.max_det]
642
+ boxes, scores, cls_ids = boxes[top], scores[top], cls_ids[top]
643
+ if len(boxes) > 1:
644
+ boxes, scores, cls_ids = self._cross_class_dedup_op(
645
+ boxes, scores, cls_ids, self.cross_iou_thresh
646
+ )
647
+ else:
648
+ # Default: per-class hard NMS -> cap -> cross-class dedup
649
+ boxes, scores, cls_ids = self._per_view_pipeline(boxes, scores, cls_ids)
650
 
651
+ results: list[BoundingBox] = []
652
+ for box, conf, cls_id in zip(boxes, scores, cls_ids):
653
+ x1, y1, x2, y2 = box.tolist()
654
+
655
+ if x2 <= x1 or y2 <= y1:
656
+ continue
657
+
658
+ results.append(
659
+ BoundingBox(
660
+ x1=int(math.floor(x1)),
661
+ y1=int(math.floor(y1)),
662
+ x2=int(math.ceil(x2)),
663
+ y2=int(math.ceil(y2)),
664
+ cls_id=int(cls_id),
665
+ conf=float(conf),
666
+ )
667
+ )
668
+
669
+ return results
670
+
671
+ def _decode_raw_yolo(
672
+ self,
673
+ preds: np.ndarray,
674
+ ratio: float,
675
+ pad: tuple[float, float],
676
+ orig_size: tuple[int, int],
677
+ ) -> list[BoundingBox]:
678
+ """
679
+ Fallback path for raw YOLO predictions.
680
+ Supports common layouts:
681
+ - [1, C, N]
682
+ - [1, N, C]
683
+ """
684
+ if preds.ndim != 3:
685
+ raise ValueError(f"Unexpected raw ONNX output shape: {preds.shape}")
686
+
687
+ if preds.shape[0] != 1:
688
+ raise ValueError(f"Unexpected batch dimension in raw output: {preds.shape}")
689
+
690
+ preds = preds[0]
691
+
692
+ # Normalize to [N, C]
693
+ if preds.shape[0] <= 16 and preds.shape[1] > preds.shape[0]:
694
+ preds = preds.T
695
+
696
+ if preds.ndim != 2 or preds.shape[1] < 5:
697
+ raise ValueError(f"Unexpected normalized raw output shape: {preds.shape}")
698
+
699
+ boxes_xywh = preds[:, :4].astype(np.float32)
700
+ cls_part = preds[:, 4:].astype(np.float32)
701
+
702
+ if cls_part.shape[1] == 1:
703
+ scores = cls_part[:, 0]
704
+ cls_ids = np.zeros(len(scores), dtype=np.int32)
705
+ else:
706
+ cls_ids = np.argmax(cls_part, axis=1).astype(np.int32)
707
+ scores = cls_part[np.arange(len(cls_part)), cls_ids]
708
+ cls_ids = self.cls_remap[cls_ids]
709
+
710
+ # Per-class confidence filter with rescue (replaces scalar threshold)
711
+ keep = self._conf_filter_mask(scores, cls_ids)
712
+ boxes_xywh = boxes_xywh[keep]
713
+ scores = scores[keep]
714
+ cls_ids = cls_ids[keep]
715
+ if len(boxes_xywh) == 0:
716
+ return []
717
+
718
+ boxes = self._xywh_to_xyxy(boxes_xywh)
719
+
720
+ # Order matches fire001 / _decode_final_dets:
721
+ # unscale -> clip -> sanity filter -> per-view pipeline (NMS, cap, cross-class dedup).
722
+ pad_w, pad_h = pad
723
  orig_w, orig_h = orig_size
724
+ boxes[:, [0, 2]] -= pad_w
725
+ boxes[:, [1, 3]] -= pad_h
726
+ boxes /= ratio
727
+ boxes = self._clip_boxes(boxes, (orig_w, orig_h))
728
+
729
+ boxes, scores, cls_ids = self._filter_sane_boxes(
730
+ boxes, scores, cls_ids, (orig_w, orig_h)
731
+ )
732
+ if len(boxes) == 0:
733
+ return []
734
+
735
+ boxes, scores, cls_ids = self._per_view_pipeline(boxes, scores, cls_ids)
736
+
737
+ results: list[BoundingBox] = []
738
  for box, conf, cls_id in zip(boxes, scores, cls_ids):
739
  x1, y1, x2, y2 = box.tolist()
740
+
741
  if x2 <= x1 or y2 <= y1:
742
  continue
743
+
744
  results.append(
745
  BoundingBox(
746
+ x1=int(math.floor(x1)),
747
+ y1=int(math.floor(y1)),
748
+ x2=int(math.ceil(x2)),
749
+ y2=int(math.ceil(y2)),
750
  cls_id=int(cls_id),
751
+ conf=float(conf),
752
  )
753
  )
754
+
755
  return results
756
 
757
+ def _postprocess(
758
+ self,
759
+ output: np.ndarray,
760
+ ratio: float,
761
+ pad: tuple[float, float],
762
+ orig_size: tuple[int, int],
763
+ ) -> list[BoundingBox]:
764
+ """
765
+ Prefer final detections first.
766
+ Fallback to raw decode only if needed.
767
+ """
768
+ # final detections: [N,6]
769
+ if output.ndim == 2 and output.shape[1] >= 6:
770
+ return self._decode_final_dets(output, ratio, pad, orig_size)
771
+
772
+ # final detections: [1,N,6]
773
+ if output.ndim == 3 and output.shape[0] == 1 and output.shape[2] == 6:
774
+ return self._decode_final_dets(output, ratio, pad, orig_size)
775
+
776
+ # fallback raw decode
777
+ return self._decode_raw_yolo(output, ratio, pad, orig_size)
778
+
779
+ def _predict_single(self, image: np.ndarray) -> list[BoundingBox]:
780
+ if image is None:
781
+ raise ValueError("Input image is None")
782
+ if not isinstance(image, np.ndarray):
783
+ raise TypeError(f"Input is not numpy array: {type(image)}")
784
+ if image.ndim != 3:
785
+ raise ValueError(f"Expected HWC image, got shape={image.shape}")
786
+ if image.shape[0] <= 0 or image.shape[1] <= 0:
787
+ raise ValueError(f"Invalid image shape={image.shape}")
788
+ if image.shape[2] != 3:
789
+ raise ValueError(f"Expected 3 channels, got shape={image.shape}")
790
+
791
+ if image.dtype != np.uint8:
792
+ image = image.astype(np.uint8)
793
+
794
+ input_tensor, ratio, pad, orig_size = self._preprocess(image)
795
+
796
+ expected_shape = (1, 3, self.input_height, self.input_width)
797
+ if input_tensor.shape != expected_shape:
798
+ raise ValueError(
799
+ f"Bad input tensor shape={input_tensor.shape}, expected={expected_shape}"
800
+ )
801
+
802
+ outputs = self.session.run(self.output_names, {self.input_name: input_tensor})
803
+ det_output = outputs[0]
804
+ return self._postprocess(det_output, ratio, pad, orig_size)
805
+
806
+ def _predict_tta(self, image: np.ndarray) -> list[BoundingBox]:
807
+ """Horizontal-flip TTA.
808
+
809
+ Strategy (ported from fire001):
810
+ 1. Predict on original and on flipped image.
811
+ 2. Map flipped boxes back to original coordinates.
812
+ 3. Per-class hard NMS on the union.
813
+ 4. For each kept box, compute the max SAME-CLASS score across the
814
+ FULL union -- a high-confidence flipped detection raises a
815
+ borderline original one, but never one of a different class.
816
+ 5. Cross-class dedup to suppress same-physical-object multi-class.
817
+ """
818
+ boxes_orig = self._predict_single(image)
819
+
820
+ flipped = cv2.flip(image, 1)
821
+ boxes_flip = self._predict_single(flipped)
822
+
823
+ w = image.shape[1]
824
  boxes_flip = [
825
+ BoundingBox(
826
+ x1=w - b.x2, y1=b.y1, x2=w - b.x1, y2=b.y2,
827
+ cls_id=b.cls_id, conf=b.conf,
828
+ )
829
  for b in boxes_flip
830
  ]
831
+
832
+ all_boxes = boxes_orig + boxes_flip
833
+ if len(all_boxes) == 0:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
834
  return []
835
 
836
+ coords = np.array(
837
+ [[b.x1, b.y1, b.x2, b.y2] for b in all_boxes], dtype=np.float32
838
+ )
839
  scores = np.array([b.conf for b in all_boxes], dtype=np.float32)
840
  cls_ids = np.array([b.cls_id for b in all_boxes], dtype=np.int32)
841
 
842
+ hard_keep = self._per_class_hard_nms(coords, scores, cls_ids, self.iou_thres)
 
843
  if len(hard_keep) == 0:
844
  return []
 
845
  if len(hard_keep) > self.max_det:
846
  top = np.argsort(-scores[hard_keep])[: self.max_det]
847
  hard_keep = hard_keep[top]
848
+
849
+ # Class-aware cluster-max score boost (fixes the silent cross-class
850
+ # leak in the previous _max_score_per_cluster).
851
  boosted = self._max_score_per_cluster(
852
  coords[hard_keep], cls_ids[hard_keep],
853
+ coords, scores, cls_ids, self.iou_thres,
854
  )
855
 
856
  kept_coords = coords[hard_keep]
857
  kept_cls = cls_ids[hard_keep]
 
 
858
  if len(kept_coords) > 1:
859
  kept_coords, boosted, kept_cls = self._cross_class_dedup_op(
860
  kept_coords, boosted, kept_cls, self.cross_iou_thresh
861
  )
862
 
863
+ return [
864
+ BoundingBox(
865
+ x1=int(math.floor(kept_coords[j, 0])),
866
+ y1=int(math.floor(kept_coords[j, 1])),
867
+ x2=int(math.ceil(kept_coords[j, 2])),
868
+ y2=int(math.ceil(kept_coords[j, 3])),
869
+ cls_id=int(kept_cls[j]),
870
+ conf=float(boosted[j]),
 
 
 
 
 
 
871
  )
872
+ for j in range(len(kept_coords))
873
+ ]
874
 
 
 
875
  def predict_batch(
876
  self,
877
  batch_images: list[ndarray],
 
879
  n_keypoints: int,
880
  ) -> list[TVFrameResult]:
881
  results: list[TVFrameResult] = []
882
+
883
+ for frame_number_in_batch, image in enumerate(batch_images):
884
  try:
885
+ if self.use_tta:
886
+ boxes = self._predict_tta(image)
887
+ else:
888
+ boxes = self._predict_single(image)
889
  except Exception as e:
890
+ print(f"⚠️ Inference failed for frame {offset + frame_number_in_batch}: {e}")
891
  boxes = []
892
+
893
  results.append(
894
  TVFrameResult(
895
+ frame_id=offset + frame_number_in_batch,
896
  boxes=boxes,
897
+ keypoints=[(0, 0) for _ in range(max(0, int(n_keypoints)))],
898
  )
899
  )
900
+
901
  return results
weights.onnx CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5eff40e23f79ec4d26d8639de704fdc432abab52f0fc44ec908037e9a2316824
3
- size 20833918
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7d0089998a8868844db5012f76470768e2cc92eeee92a2c22e84c46fd4109ea5
3
+ size 19287011