SuperBitDev commited on
Commit
96cabc4
·
verified ·
1 Parent(s): 30ebcaf

Upload folder using huggingface_hub

Browse files
Files changed (3) hide show
  1. chute_config.yml +0 -1
  2. miner.py +452 -428
  3. weights.onnx +2 -2
chute_config.yml CHANGED
@@ -8,7 +8,6 @@ Image:
8
  NodeSelector:
9
  gpu_count: 1
10
  min_vram_gb_per_gpu: 16
11
- max_hourly_price_per_gpu: 2
12
  include:
13
  - pro_6000
14
 
 
8
  NodeSelector:
9
  gpu_count: 1
10
  min_vram_gb_per_gpu: 16
 
11
  include:
12
  - pro_6000
13
 
miner.py CHANGED
@@ -1,5 +1,4 @@
1
  from pathlib import Path
2
- import math
3
 
4
  import cv2
5
  import numpy as np
@@ -24,20 +23,74 @@ class TVFrameResult(BaseModel):
24
 
25
 
26
  class Miner:
27
- def __init__(self,
28
- path_hf_repo: Path
29
- ) -> None:
30
- model_path = self._resolve_model_path(path_hf_repo)
31
- # car-wash element classes cls_id order MUST match element `objects`
32
- # (0=broom, 1=drainage gate, 2=nozzle, 3=track). This is the canonical
33
- # order every downstream consumer (validator, BoundingBox.cls_id) sees.
34
- self.class_names = ["broom", "drainage gate", "nozzle", "track"]
35
- # FALLBACK model-emit order: the authoritative order is read from the
36
- # ONNX `names` metadata after the session is created (embedded by
37
- # Ultralytics at export, ships inside weights.onnx), so a retrained
38
- # model with a different class order is remapped correctly without
39
- # code changes. This list is used only when metadata is missing.
40
- self._model_class_order = ["broom", "drainage gate", "nozzle", "track"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
41
  print("ORT version:", ort.__version__)
42
 
43
  try:
@@ -58,11 +111,9 @@ class Miner:
58
  self.session = ort.InferenceSession(
59
  str(model_path),
60
  sess_options=sess_options,
61
- providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
62
  )
63
- print("✅ Created ORT session with preferred CUDA provider list")
64
  except Exception as e:
65
- print(f"⚠️ CUDA session creation failed, falling back to CPU: {e}")
66
  self.session = ort.InferenceSession(
67
  str(model_path),
68
  sess_options=sess_options,
@@ -73,8 +124,10 @@ class Miner:
73
 
74
  # Build cls_remap: for each model-emit index i,
75
  # cls_remap[i] = self.class_names.index(model_class_order[i])
76
- # The model-side order comes from the ONNX metadata when available,
77
- # else falls back to the static _model_class_order.
 
 
78
  model_class_order = self._read_model_class_order()
79
  if model_class_order is None:
80
  model_class_order = list(self._model_class_order)
@@ -82,12 +135,12 @@ class Miner:
82
  else:
83
  print(f"cls order: from ONNX metadata {model_class_order}")
84
  self.cls_remap = np.array(
85
- [self.class_names.index(n) for n in model_class_order], dtype=np.int32
 
86
  )
87
 
88
  for inp in self.session.get_inputs():
89
  print("INPUT:", inp.name, inp.shape, inp.type)
90
-
91
  for out in self.session.get_outputs():
92
  print("OUTPUT:", out.name, out.shape, out.type)
93
 
@@ -95,56 +148,32 @@ class Miner:
95
  self.output_names = [output.name for output in self.session.get_outputs()]
96
  self.input_shape = self.session.get_inputs()[0].shape
97
 
98
- # Match the ONNX input dtype (this export is FP16 -> needs float16 input).
99
- input_type = self.session.get_inputs()[0].type
100
- self.np_dtype = np.float16 if "float16" in input_type else np.float32
101
- print(f"✅ ONNX input dtype: {input_type} -> numpy {self.np_dtype}")
102
-
103
- # ONNX is fixed-size 1408x1408 (v1 export); read actual shape to be safe.
104
  self.input_height = self._safe_dim(self.input_shape[2], default=1280)
105
  self.input_width = self._safe_dim(self.input_shape[3], default=1280)
106
 
107
- # Tuned for validator scoring (pillars: 0.6*map50 + 0.4*false_positive).
108
- # All values below are the measured optimum of a full TTA-off grid sweep
109
- # on the validator-style val split (tune_miner.py, car-wash-49-val1024
110
- # val, ALL 2476 crops, against the synthetic-crop model; composite
111
- # 0.8716 -> 0.8731) -- re-run the sweep after any retrain.
112
- self.iou_thres = 0.5 # Per-class NMS IoU; lower = stricter dedup
113
- self.cross_iou_thresh = 0.9 # Cross-class dedup IoU (suppress same physical object firing multiple classes)
114
- self.max_det = 200
115
- # TTA = a 2nd (flipped) forward pass. Doubles latency; off for the
116
- # CPU latency gate. Re-enable only if the latency budget allows.
117
  self.use_tta = False
118
-
119
- # conf thresholds: broom=0.38 drainage gate=0.45 nozzle=0.30 track=0.60
120
- # Per-class confidence thresholds.
121
- # Indexed by class_names order: [broom, drainage gate, nozzle, track].
122
- # Values are the 2476-crop TTA-off sweep optimum on the synthetic-crop
123
- # model: lowering drainage (0.30->0.22) and nozzle (0.45->0.38) recovers
124
- # recall (map50 0.836->0.847) while FP actually drops (0.925->0.913).
125
- self._conf_thres_array = np.array(
126
- [0.30, 0.30, 0.40, 0.30], dtype=np.float32
127
- )
128
- # Per-class rescue bonus: when a class has ZERO boxes passing the
129
- # threshold in a frame, its top-1 candidate is admitted when its score
130
- # is at least (per-class threshold - per-class bonus).
131
- # DISABLED (all zeros): the 2476-crop sweep (rescue_bonus=False won)
132
- # confirmed rescue admits more false positives than true positives
133
- # under the validator's FP pillar.
134
- self._bonus_array = np.array(
135
- [0.05, 0.05, 0.1, 0.05], dtype=np.float32
136
- )
137
-
138
- # Box sanity filter — kept loose: car-wash `nozzle` boxes are tiny
139
- # (GT median ~290 px², smallest ~32 px²). Fire's 14x14/min_side 8
140
- # would delete valid nozzles, so thresholds are dropped here.
141
- self.min_box_area = 4 * 4 # 16 px²
142
- self.min_side = 3
143
- self.max_aspect_ratio = 12.0
144
 
145
  print(f"✅ ONNX model loaded from: {model_path}")
146
  print(f"✅ ONNX providers: {self.session.get_providers()}")
147
  print(f"✅ ONNX input: name={self.input_name}, shape={self.input_shape}")
 
 
 
 
 
 
 
148
 
149
  self._warmup()
150
 
@@ -157,50 +186,16 @@ class Miner:
157
  except Exception as e:
158
  print(f"⚠️ warmup skipped: {e}")
159
 
160
- def __repr__(self) -> str:
161
- return (
162
- f"ONNXRuntime(session={type(self.session).__name__}, "
163
- f"providers={self.session.get_providers()})"
164
- )
165
-
166
- @staticmethod
167
- def _safe_dim(value, default: int) -> int:
168
- return value if isinstance(value, int) and value > 0 else default
169
-
170
- @staticmethod
171
- def _resolve_model_path(repo: Path) -> Path:
172
- """Locate the ONNX model in the repo dir.
173
-
174
- Prefers weights.onnx (FP16/FP32 export), then weights_int8.onnx (the
175
- training script's INT8-quantized export -- works as-is: quantization
176
- preserves the Ultralytics metadata and QDQ models take regular fp32
177
- input), then any other .onnx file. INT8 is the fallback when the FP16
178
- export exceeds the 30 MB deployment limit (e.g. yolo26m).
179
- """
180
- for name in ("weights.onnx", "weights_int8.onnx"):
181
- p = repo / name
182
- if p.exists():
183
- if name != "weights.onnx":
184
- print(f"model: weights.onnx not found, using {name}")
185
- return p
186
- candidates = sorted(repo.glob("*.onnx"))
187
- if candidates:
188
- print(f"model: using {candidates[0].name}")
189
- return candidates[0]
190
- return repo / "weights.onnx" # let session creation raise the error
191
-
192
- def _read_model_class_order(self) -> list[str] | None:
193
  """Read the model's class order from Ultralytics ONNX metadata.
194
-
195
- Returns the class names ordered by model-emit index, or None when
196
- metadata is missing/unparsable or doesn't match `class_names` as a
197
- set (in which case the static _model_class_order fallback is used).
198
- """
199
  try:
200
  import ast
201
 
202
  meta = self.session.get_modelmeta().custom_metadata_map
203
- names = ast.literal_eval(meta["names"]) # e.g. {0: 'broom', ...}
204
  if isinstance(names, dict):
205
  order = [str(names[i]) for i in sorted(names)]
206
  else:
@@ -216,19 +211,22 @@ class Miner:
216
  return None
217
  return order
218
 
 
 
 
 
 
 
 
 
 
 
219
  def _letterbox(
220
  self,
221
  image: ndarray,
222
  new_shape: tuple[int, int],
223
  color=(114, 114, 114),
224
  ) -> tuple[ndarray, float, tuple[float, float]]:
225
- """
226
- Resize with unchanged aspect ratio and pad to target shape.
227
- Returns:
228
- padded_image,
229
- ratio,
230
- (pad_w, pad_h) # half-padding
231
- """
232
  h, w = image.shape[:2]
233
  new_w, new_h = new_shape
234
 
@@ -240,10 +238,8 @@ class Miner:
240
  interp = cv2.INTER_CUBIC if ratio > 1.0 else cv2.INTER_LINEAR
241
  image = cv2.resize(image, (resized_w, resized_h), interpolation=interp)
242
 
243
- dw = new_w - resized_w
244
- dh = new_h - resized_h
245
- dw /= 2.0
246
- dh /= 2.0
247
 
248
  left = int(round(dw - 0.1))
249
  right = int(round(dw + 0.1))
@@ -251,38 +247,23 @@ class Miner:
251
  bottom = int(round(dh + 0.1))
252
 
253
  padded = cv2.copyMakeBorder(
254
- image,
255
- top,
256
- bottom,
257
- left,
258
- right,
259
- borderType=cv2.BORDER_CONSTANT,
260
- value=color,
261
  )
262
  return padded, ratio, (dw, dh)
263
 
264
  def _preprocess(
265
  self, image: ndarray
266
  ) -> tuple[np.ndarray, float, tuple[float, float], tuple[int, int]]:
267
- """
268
- Preprocess for fixed-size ONNX export:
269
- - enhance image quality (CLAHE, denoise, sharpen)
270
- - letterbox to model input size
271
- - BGR -> RGB
272
- - normalize to [0,1]
273
- - HWC -> NCHW float32
274
- """
275
  orig_h, orig_w = image.shape[:2]
276
-
277
  img, ratio, pad = self._letterbox(
278
  image, (self.input_width, self.input_height)
279
  )
280
- img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
281
- img = (img.astype(np.float32) / 255.0)
282
- img = np.transpose(img, (2, 0, 1))[None, ...]
283
- img = np.ascontiguousarray(img, dtype=self.np_dtype)
284
-
285
- return img, ratio, pad, (orig_w, orig_h)
286
 
287
  @staticmethod
288
  def _clip_boxes(boxes: np.ndarray, image_size: tuple[int, int]) -> np.ndarray:
@@ -302,6 +283,52 @@ class Miner:
302
  out[:, 3] = boxes[:, 1] + boxes[:, 3] / 2.0
303
  return out
304
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
305
  def _soft_nms(
306
  self,
307
  boxes: np.ndarray,
@@ -309,106 +336,35 @@ class Miner:
309
  sigma: float = 0.5,
310
  score_thresh: float = 0.01,
311
  ) -> tuple[np.ndarray, np.ndarray]:
312
- """
313
- Soft-NMS: Gaussian decay of overlapping scores instead of hard removal.
314
- Returns (kept_original_indices, updated_scores).
315
- """
316
  N = len(boxes)
317
  if N == 0:
318
  return np.array([], dtype=np.intp), np.array([], dtype=np.float32)
319
-
320
  boxes = boxes.astype(np.float32, copy=True)
321
  scores = scores.astype(np.float32, copy=True)
322
  order = np.arange(N)
323
-
324
  for i in range(N):
325
  max_pos = i + int(np.argmax(scores[i:]))
326
  boxes[[i, max_pos]] = boxes[[max_pos, i]]
327
  scores[[i, max_pos]] = scores[[max_pos, i]]
328
  order[[i, max_pos]] = order[[max_pos, i]]
329
-
330
  if i + 1 >= N:
331
  break
332
-
333
  xx1 = np.maximum(boxes[i, 0], boxes[i + 1:, 0])
334
  yy1 = np.maximum(boxes[i, 1], boxes[i + 1:, 1])
335
  xx2 = np.minimum(boxes[i, 2], boxes[i + 1:, 2])
336
  yy2 = np.minimum(boxes[i, 3], boxes[i + 1:, 3])
337
  inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
338
-
339
  area_i = max(0.0, float(
340
- (boxes[i, 2] - boxes[i, 0]) * (boxes[i, 3] - boxes[i, 1])
341
- ))
342
- areas_j = (
343
- np.maximum(0.0, boxes[i + 1:, 2] - boxes[i + 1:, 0])
344
- * np.maximum(0.0, boxes[i + 1:, 3] - boxes[i + 1:, 1])
345
- )
346
  iou = inter / (area_i + areas_j - inter + 1e-7)
347
  scores[i + 1:] *= np.exp(-(iou ** 2) / sigma)
348
-
349
  mask = scores > score_thresh
350
  return order[mask], scores[mask]
351
 
352
- @staticmethod
353
- def _hard_nms(
354
- boxes: np.ndarray,
355
- scores: np.ndarray,
356
- iou_thresh: float,
357
- ) -> np.ndarray:
358
- """
359
- Standard NMS: keep one box per overlapping cluster (the one with highest score).
360
- Returns indices of kept boxes (into the boxes/scores arrays).
361
- """
362
- N = len(boxes)
363
- if N == 0:
364
- return np.array([], dtype=np.intp)
365
- boxes = np.asarray(boxes, dtype=np.float32)
366
- scores = np.asarray(scores, dtype=np.float32)
367
- order = np.argsort(scores)[::-1]
368
- keep: list[int] = []
369
- suppressed = np.zeros(N, dtype=bool)
370
- for i in range(N):
371
- idx = order[i]
372
- if suppressed[idx]:
373
- continue
374
- keep.append(idx)
375
- bi = boxes[idx]
376
- for k in range(i + 1, N):
377
- jdx = order[k]
378
- if suppressed[jdx]:
379
- continue
380
- bj = boxes[jdx]
381
- xx1 = max(bi[0], bj[0])
382
- yy1 = max(bi[1], bj[1])
383
- xx2 = min(bi[2], bj[2])
384
- yy2 = min(bi[3], bj[3])
385
- inter = max(0.0, xx2 - xx1) * max(0.0, yy2 - yy1)
386
- area_i = (bi[2] - bi[0]) * (bi[3] - bi[1])
387
- area_j = (bj[2] - bj[0]) * (bj[3] - bj[1])
388
- iou = inter / (area_i + area_j - inter + 1e-7)
389
- if iou > iou_thresh:
390
- suppressed[jdx] = True
391
- return np.array(keep)
392
-
393
- def _per_class_hard_nms(
394
- self,
395
- boxes: np.ndarray,
396
- scores: np.ndarray,
397
- cls_ids: np.ndarray,
398
- iou_thresh: float,
399
- ) -> np.ndarray:
400
- """Hard NMS applied independently per class."""
401
- if len(boxes) == 0:
402
- return np.array([], dtype=np.intp)
403
- all_keep: list[int] = []
404
- for c in np.unique(cls_ids):
405
- mask = cls_ids == c
406
- indices = np.where(mask)[0]
407
- keep = self._hard_nms(boxes[mask], scores[mask], iou_thresh)
408
- all_keep.extend(indices[keep].tolist())
409
- all_keep.sort()
410
- return np.array(all_keep, dtype=np.intp)
411
-
412
  def _per_class_soft_nms(
413
  self,
414
  boxes: np.ndarray,
@@ -417,60 +373,64 @@ class Miner:
417
  sigma: float = 0.5,
418
  score_thresh: float = 0.01,
419
  ) -> tuple[np.ndarray, np.ndarray]:
420
- """Soft NMS applied independently per class."""
421
  if len(boxes) == 0:
422
  return np.array([], dtype=np.intp), np.array([], dtype=np.float32)
423
  all_keep: list[int] = []
424
  all_scores: list[float] = []
425
  for c in np.unique(cls_ids):
426
- mask = cls_ids == c
427
- indices = np.where(mask)[0]
428
- keep, updated = self._soft_nms(boxes[mask], scores[mask], sigma, score_thresh)
429
  for k, s in zip(keep, updated):
430
- all_keep.append(int(indices[k]))
431
- all_scores.append(float(s))
432
  if not all_keep:
433
  return np.array([], dtype=np.intp), np.array([], dtype=np.float32)
434
  return np.array(all_keep, dtype=np.intp), np.array(all_scores, dtype=np.float32)
435
 
436
- def _filter_sane_boxes(
437
  self,
438
  boxes: np.ndarray,
439
  scores: np.ndarray,
440
  cls_ids: np.ndarray,
441
- orig_size: tuple[int, int],
442
  ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
443
- """Filter out tiny, degenerate, or implausible boxes (common FP)."""
444
- if len(boxes) == 0:
 
 
 
 
 
 
 
445
  return boxes, scores, cls_ids
446
- orig_w, orig_h = orig_size
447
- image_area = float(orig_w * orig_h)
448
- keep = []
449
- for i, box in enumerate(boxes):
450
- x1, y1, x2, y2 = box.tolist()
451
- bw = x2 - x1
452
- bh = y2 - y1
453
- if bw <= 0 or bh <= 0:
454
- continue
455
- if bw < self.min_side or bh < self.min_side:
456
- continue
457
- area = bw * bh
458
- if area < self.min_box_area:
459
- continue
460
- if area > 0.95 * image_area:
461
- continue
462
- ar = max(bw / max(bh, 1e-6), bh / max(bw, 1e-6))
463
- if ar > self.max_aspect_ratio:
464
  continue
465
- keep.append(i)
466
- if not keep:
467
- return (
468
- np.empty((0, 4), dtype=np.float32),
469
- np.empty((0,), dtype=np.float32),
470
- np.empty((0,), dtype=np.int32),
471
- )
472
- k = np.array(keep, dtype=np.intp)
473
- return boxes[k], scores[k], cls_ids[k]
 
 
 
 
 
474
 
475
  @staticmethod
476
  def _max_score_per_cluster(
@@ -482,11 +442,9 @@ class Miner:
482
  iou_thresh: float,
483
  ) -> np.ndarray:
484
  """For each kept (post-NMS) box, return the max score over the FULL
485
- candidate set among SAME-CLASS boxes with IoU >= iou_thresh.
486
-
487
- The previous version omitted the same-class constraint, which let a
488
- confident broom raise the score of a coincident nozzle (or vice
489
- versa) under TTA. That's a silent FP booster and is fixed here.
490
  """
491
  n = len(post_boxes)
492
  if n == 0:
@@ -512,8 +470,7 @@ class Miner:
512
  ) -> np.ndarray:
513
  """Boolean keep-mask: score >= per-class threshold, with a per-class
514
  rescue -- if a class has zero boxes passing, admit its top-1 candidate
515
- when its score >= (per-class threshold - per-class bonus).
516
- """
517
  if len(scores) == 0:
518
  return np.zeros(0, dtype=bool)
519
  thr = self._conf_thres_array[cls_ids]
@@ -531,50 +488,44 @@ class Miner:
531
  keep[top] = True
532
  return keep
533
 
534
- def _cross_class_dedup_op(
535
  self,
536
  boxes: np.ndarray,
537
  scores: np.ndarray,
538
  cls_ids: np.ndarray,
539
- iou_thresh: float,
540
  ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
541
- """Remove near-duplicate boxes across classes.
542
-
543
- Order candidates by (score - per_class_threshold) margin, then by area;
544
- keep the highest, suppress every other box with IoU > iou_thresh. For
545
- car-wash this kills the common failure where water spray makes the
546
- model fire both `nozzle` and `track` on the same patch, or where a
547
- broom handle overlaps a drainage-gate detection.
548
- """
549
- n = len(boxes)
550
- if n <= 1:
551
  return boxes, scores, cls_ids
552
- boxes = np.asarray(boxes, dtype=np.float32)
553
- scores = np.asarray(scores, dtype=np.float32)
554
- cls_ids = np.asarray(cls_ids, dtype=np.int32)
555
- areas = (np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) *
556
- np.maximum(0.0, boxes[:, 3] - boxes[:, 1]))
557
- margins = scores - self._conf_thres_array[cls_ids]
558
- order = np.lexsort((-areas, -margins))
559
- suppressed = np.zeros(n, dtype=bool)
560
- keep: list[int] = []
561
- for i in order:
562
- if suppressed[i]:
563
  continue
564
- keep.append(int(i))
565
- bi = boxes[i]
566
- xx1 = np.maximum(bi[0], boxes[:, 0])
567
- yy1 = np.maximum(bi[1], boxes[:, 1])
568
- xx2 = np.minimum(bi[2], boxes[:, 2])
569
- yy2 = np.minimum(bi[3], boxes[:, 3])
570
- inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
571
- a_i = max(1e-7, float((bi[2] - bi[0]) * (bi[3] - bi[1])))
572
- iou = inter / (a_i + areas - inter + 1e-7)
573
- dup = iou > iou_thresh
574
- dup[i] = False
575
- suppressed |= dup
576
- keep_idx = np.array(keep, dtype=np.intp)
577
- return boxes[keep_idx], scores[keep_idx], cls_ids[keep_idx]
 
 
 
 
 
578
 
579
  def _per_view_pipeline(
580
  self,
@@ -582,10 +533,16 @@ class Miner:
582
  scores: np.ndarray,
583
  cls_ids: np.ndarray,
584
  ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
585
- """Per-view post-processing: per-class NMS -> cap -> cross-class dedup."""
586
  if len(boxes) > 1:
587
- keep = self._per_class_hard_nms(boxes, scores, cls_ids, self.iou_thres)
588
- boxes, scores, cls_ids = boxes[keep], scores[keep], cls_ids[keep]
 
 
 
 
 
 
589
  if len(scores) > self.max_det:
590
  top = np.argsort(-scores)[: self.max_det]
591
  boxes, scores, cls_ids = boxes[top], scores[top], cls_ids[top]
@@ -595,22 +552,53 @@ class Miner:
595
  )
596
  return boxes, scores, cls_ids
597
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
598
  def _decode_final_dets(
599
  self,
600
  preds: np.ndarray,
601
  ratio: float,
602
  pad: tuple[float, float],
603
  orig_size: tuple[int, int],
604
- apply_optional_dedup: bool = False,
605
  ) -> list[BoundingBox]:
606
- """
607
- Primary path:
608
- expected output rows like [x1, y1, x2, y2, conf, cls_id]
609
- in letterboxed input coordinates.
610
- """
611
  if preds.ndim == 3 and preds.shape[0] == 1:
612
  preds = preds[0]
613
-
614
  if preds.ndim != 2 or preds.shape[1] < 6:
615
  raise ValueError(f"Unexpected ONNX final-det output shape: {preds.shape}")
616
 
@@ -619,66 +607,27 @@ class Miner:
619
  cls_ids = preds[:, 5].astype(np.int32)
620
  cls_ids = self.cls_remap[cls_ids]
621
 
622
- # Per-class confidence filter with rescue (replaces scalar threshold)
623
  keep = self._conf_filter_mask(scores, cls_ids)
624
  boxes = boxes[keep]
625
  scores = scores[keep]
626
  cls_ids = cls_ids[keep]
627
-
628
  if len(boxes) == 0:
629
  return []
630
 
631
  pad_w, pad_h = pad
632
- orig_w, orig_h = orig_size
633
-
634
- # reverse letterbox
635
  boxes[:, [0, 2]] -= pad_w
636
  boxes[:, [1, 3]] -= pad_h
637
  boxes /= ratio
638
- boxes = self._clip_boxes(boxes, (orig_w, orig_h))
639
 
640
- # Box sanity filter (reduces FP)
641
  boxes, scores, cls_ids = self._filter_sane_boxes(
642
  boxes, scores, cls_ids, orig_size
643
  )
644
  if len(boxes) == 0:
645
  return []
646
 
647
- if apply_optional_dedup and len(boxes) > 1:
648
- # Soft-NMS path preserved as a tunable option; default below.
649
- keep_idx, scores = self._per_class_soft_nms(boxes, scores, cls_ids)
650
- boxes = boxes[keep_idx]
651
- cls_ids = cls_ids[keep_idx]
652
- if len(scores) > self.max_det:
653
- top = np.argsort(-scores)[: self.max_det]
654
- boxes, scores, cls_ids = boxes[top], scores[top], cls_ids[top]
655
- if len(boxes) > 1:
656
- boxes, scores, cls_ids = self._cross_class_dedup_op(
657
- boxes, scores, cls_ids, self.cross_iou_thresh
658
- )
659
- else:
660
- # Default: per-class hard NMS -> cap -> cross-class dedup
661
- boxes, scores, cls_ids = self._per_view_pipeline(boxes, scores, cls_ids)
662
-
663
- results: list[BoundingBox] = []
664
- for box, conf, cls_id in zip(boxes, scores, cls_ids):
665
- x1, y1, x2, y2 = box.tolist()
666
-
667
- if x2 <= x1 or y2 <= y1:
668
- continue
669
-
670
- results.append(
671
- BoundingBox(
672
- x1=int(math.floor(x1)),
673
- y1=int(math.floor(y1)),
674
- x2=int(math.ceil(x2)),
675
- y2=int(math.ceil(y2)),
676
- cls_id=int(cls_id),
677
- conf=float(conf),
678
- )
679
- )
680
-
681
- return results
682
 
683
  def _decode_raw_yolo(
684
  self,
@@ -687,30 +636,17 @@ class Miner:
687
  pad: tuple[float, float],
688
  orig_size: tuple[int, int],
689
  ) -> list[BoundingBox]:
690
- """
691
- Fallback path for raw YOLO predictions.
692
- Supports common layouts:
693
- - [1, C, N]
694
- - [1, N, C]
695
- """
696
- if preds.ndim != 3:
697
  raise ValueError(f"Unexpected raw ONNX output shape: {preds.shape}")
698
-
699
- if preds.shape[0] != 1:
700
- raise ValueError(f"Unexpected batch dimension in raw output: {preds.shape}")
701
-
702
  preds = preds[0]
703
-
704
- # Normalize to [N, C]
705
  if preds.shape[0] <= 16 and preds.shape[1] > preds.shape[0]:
706
  preds = preds.T
707
-
708
  if preds.ndim != 2 or preds.shape[1] < 5:
709
- raise ValueError(f"Unexpected normalized raw output shape: {preds.shape}")
710
 
711
  boxes_xywh = preds[:, :4].astype(np.float32)
712
  cls_part = preds[:, 4:].astype(np.float32)
713
-
714
  if cls_part.shape[1] == 1:
715
  scores = cls_part[:, 0]
716
  cls_ids = np.zeros(len(scores), dtype=np.int32)
@@ -719,52 +655,28 @@ class Miner:
719
  scores = cls_part[np.arange(len(cls_part)), cls_ids]
720
  cls_ids = self.cls_remap[cls_ids]
721
 
722
- # Per-class confidence filter with rescue (replaces scalar threshold)
723
  keep = self._conf_filter_mask(scores, cls_ids)
724
  boxes_xywh = boxes_xywh[keep]
725
  scores = scores[keep]
726
  cls_ids = cls_ids[keep]
727
  if len(boxes_xywh) == 0:
728
  return []
729
-
730
  boxes = self._xywh_to_xyxy(boxes_xywh)
731
 
732
- # Order matches fire001 / _decode_final_dets:
733
- # unscale -> clip -> sanity filter -> per-view pipeline (NMS, cap, cross-class dedup).
734
  pad_w, pad_h = pad
735
- orig_w, orig_h = orig_size
736
  boxes[:, [0, 2]] -= pad_w
737
  boxes[:, [1, 3]] -= pad_h
738
  boxes /= ratio
739
- boxes = self._clip_boxes(boxes, (orig_w, orig_h))
740
 
741
  boxes, scores, cls_ids = self._filter_sane_boxes(
742
- boxes, scores, cls_ids, (orig_w, orig_h)
743
  )
744
  if len(boxes) == 0:
745
  return []
746
 
747
  boxes, scores, cls_ids = self._per_view_pipeline(boxes, scores, cls_ids)
748
-
749
- results: list[BoundingBox] = []
750
- for box, conf, cls_id in zip(boxes, scores, cls_ids):
751
- x1, y1, x2, y2 = box.tolist()
752
-
753
- if x2 <= x1 or y2 <= y1:
754
- continue
755
-
756
- results.append(
757
- BoundingBox(
758
- x1=int(math.floor(x1)),
759
- y1=int(math.floor(y1)),
760
- x2=int(math.ceil(x2)),
761
- y2=int(math.ceil(y2)),
762
- cls_id=int(cls_id),
763
- conf=float(conf),
764
- )
765
- )
766
-
767
- return results
768
 
769
  def _postprocess(
770
  self,
@@ -773,19 +685,10 @@ class Miner:
773
  pad: tuple[float, float],
774
  orig_size: tuple[int, int],
775
  ) -> list[BoundingBox]:
776
- """
777
- Prefer final detections first.
778
- Fallback to raw decode only if needed.
779
- """
780
- # final detections: [N,6]
781
  if output.ndim == 2 and output.shape[1] >= 6:
782
  return self._decode_final_dets(output, ratio, pad, orig_size)
783
-
784
- # final detections: [1,N,6]
785
  if output.ndim == 3 and output.shape[0] == 1 and output.shape[2] == 6:
786
  return self._decode_final_dets(output, ratio, pad, orig_size)
787
-
788
- # fallback raw decode
789
  return self._decode_raw_yolo(output, ratio, pad, orig_size)
790
 
791
  def _predict_single(self, image: np.ndarray) -> list[BoundingBox]:
@@ -799,39 +702,33 @@ class Miner:
799
  raise ValueError(f"Invalid image shape={image.shape}")
800
  if image.shape[2] != 3:
801
  raise ValueError(f"Expected 3 channels, got shape={image.shape}")
802
-
803
  if image.dtype != np.uint8:
804
  image = image.astype(np.uint8)
805
 
806
  input_tensor, ratio, pad, orig_size = self._preprocess(image)
807
-
808
- expected_shape = (1, 3, self.input_height, self.input_width)
809
- if input_tensor.shape != expected_shape:
810
  raise ValueError(
811
- f"Bad input tensor shape={input_tensor.shape}, expected={expected_shape}"
812
  )
813
 
814
  outputs = self.session.run(self.output_names, {self.input_name: input_tensor})
815
- det_output = outputs[0]
816
- return self._postprocess(det_output, ratio, pad, orig_size)
817
 
818
  def _predict_tta(self, image: np.ndarray) -> list[BoundingBox]:
819
  """Horizontal-flip TTA.
820
-
821
- Strategy (ported from fire001):
822
  1. Predict on original and on flipped image.
823
  2. Map flipped boxes back to original coordinates.
824
  3. Per-class hard NMS on the union.
825
- 4. For each kept box, compute the max SAME-CLASS score across the
826
- FULL union -- a high-confidence flipped detection raises a
827
- borderline original one, but never one of a different class.
828
  5. Cross-class dedup to suppress same-physical-object multi-class.
829
  """
830
  boxes_orig = self._predict_single(image)
831
-
832
  flipped = cv2.flip(image, 1)
833
  boxes_flip = self._predict_single(flipped)
834
-
835
  w = image.shape[1]
836
  boxes_flip = [
837
  BoundingBox(
@@ -840,9 +737,8 @@ class Miner:
840
  )
841
  for b in boxes_flip
842
  ]
843
-
844
  all_boxes = boxes_orig + boxes_flip
845
- if len(all_boxes) == 0:
846
  return []
847
 
848
  coords = np.array(
@@ -858,8 +754,6 @@ class Miner:
858
  top = np.argsort(-scores[hard_keep])[: self.max_det]
859
  hard_keep = hard_keep[top]
860
 
861
- # Class-aware cluster-max score boost (fixes the silent cross-class
862
- # leak in the previous _max_score_per_cluster).
863
  boosted = self._max_score_per_cluster(
864
  coords[hard_keep], cls_ids[hard_keep],
865
  coords, scores, cls_ids, self.iou_thres,
@@ -872,18 +766,151 @@ class Miner:
872
  kept_coords, boosted, kept_cls, self.cross_iou_thresh
873
  )
874
 
 
 
 
875
  return [
876
  BoundingBox(
877
- x1=int(math.floor(kept_coords[j, 0])),
878
- y1=int(math.floor(kept_coords[j, 1])),
879
- x2=int(math.ceil(kept_coords[j, 2])),
880
- y2=int(math.ceil(kept_coords[j, 3])),
881
  cls_id=int(kept_cls[j]),
882
  conf=float(boosted[j]),
883
  )
884
  for j in range(len(kept_coords))
885
  ]
886
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
887
  def predict_batch(
888
  self,
889
  batch_images: list[ndarray],
@@ -891,17 +918,15 @@ class Miner:
891
  n_keypoints: int,
892
  ) -> list[TVFrameResult]:
893
  results: list[TVFrameResult] = []
894
-
895
  for frame_number_in_batch, image in enumerate(batch_images):
896
  try:
897
- if self.use_tta:
898
- boxes = self._predict_tta(image)
899
- else:
900
- boxes = self._predict_single(image)
901
  except Exception as e:
902
- print(f"⚠️ Inference failed for frame {offset + frame_number_in_batch}: {e}")
 
 
 
903
  boxes = []
904
-
905
  results.append(
906
  TVFrameResult(
907
  frame_id=offset + frame_number_in_batch,
@@ -909,5 +934,4 @@ class Miner:
909
  keypoints=[(0, 0) for _ in range(max(0, int(n_keypoints)))],
910
  )
911
  )
912
-
913
- return results
 
1
  from pathlib import Path
 
2
 
3
  import cv2
4
  import numpy as np
 
23
 
24
 
25
  class Miner:
26
+ """ONNX Runtime miner for road-sign detection (single class).
27
+ Strategy (ported from offense / fire001 miner):
28
+ - per-class confidence threshold with per-class rescue bonus
29
+ - per-class hard NMS, then cross-class dedup (no-op for single class)
30
+ - horizontal-flip TTA with full-set cluster score boost
31
+ Plus: class remap, sanity-box filter tuned for small distant signs,
32
+ TTA toggle.
33
+ """
34
+
35
+ class_names = ["road_sign"]
36
+ # Order the model emits classes in -- remapped to `class_names` index.
37
+ _model_class_order = ["road_sign"]
38
+
39
+ iou_thres = 0.5
40
+ cross_iou_thresh = 0.8
41
+ max_det = 150
42
+
43
+ # Per-class confidence threshold. Road signs in this dataset are
44
+ # frequently degraded / rear-facing / partly-obscured / distant, so we
45
+ # run noticeably below the fire/smoke baseline. The validator's
46
+ # false_positive pillar = max(0, 1 - ffpi/10): we can tolerate ~2 FP per
47
+ # image and still keep that pillar above 0.8.
48
+ _conf_thres_array = np.array(
49
+ [0.33], dtype=np.float32
50
+ )
51
+ # Per-class rescue bonus. If a class has ZERO boxes passing the threshold
52
+ # in a frame, its top-1 candidate is admitted when its score is at least
53
+ # (threshold - bonus). Bumped from 0.05 -> 0.10 so a single faint sign in
54
+ # an otherwise empty frame still produces a detection (map50 recall win,
55
+ # at most one extra FP per such frame).
56
+ _bonus_array = np.array(
57
+ [0.12], dtype=np.float32
58
+ )
59
+
60
+ # Box sanity filter: drop tiny / degenerate / image-spanning / extreme
61
+ # aspect ratio boxes.
62
+ # min_box_area = 14x14 -> 14x14 is the smallest credible sign. The old
63
+ # value of 64 (8x8) silently discarded narrow
64
+ # distant signs like a 10x6 px overhead chevron.
65
+ # min_side = 3 -> matches min_box_area; anything thinner is
66
+ # almost certainly a pole or shadow false alarm.
67
+ # max_aspect_ratio = 12.0
68
+ # -> overhead destination panels and lane-assignment
69
+ # signs are very wide (long, thin rectangles);
70
+ # 8.0 was clipping legitimate detections.
71
+ min_box_area = 8 * 8
72
+ min_side = 3
73
+ max_aspect_ratio = 12.0
74
+
75
+ # Final box-size calibration. The detector + de-letterbox + integer-rounding
76
+ # pipeline emits boxes slightly larger than the object, so shrink every
77
+ # emitted box about its center by a fixed per-axis factor before output:
78
+ # new_w = w / box_shrink_w, new_h = h / box_shrink_h.
79
+ box_shrink_w = 1.027
80
+ box_shrink_h = 1.014
81
+
82
+ # Tile-based TTA: when the source image is significantly larger than the
83
+ # model input, letterboxing throws away ~1.5x of effective resolution,
84
+ # which kills small-sign recall. Splitting into overlapping horizontal
85
+ # tiles preserves native resolution on each half. Triggered only when
86
+ # source width >= tile_trigger_ratio * model_input_width to avoid wasted
87
+ # compute on already-small images.
88
+ tile_trigger_ratio = 1.4
89
+ tile_overlap_ratio = 0.20
90
+
91
+ def __init__(self, path_hf_repo: Path) -> None:
92
+ model_path = path_hf_repo / "weights.onnx"
93
+
94
  print("ORT version:", ort.__version__)
95
 
96
  try:
 
111
  self.session = ort.InferenceSession(
112
  str(model_path),
113
  sess_options=sess_options,
114
+ providers=["CPUExecutionProvider"],
115
  )
 
116
  except Exception as e:
 
117
  self.session = ort.InferenceSession(
118
  str(model_path),
119
  sess_options=sess_options,
 
124
 
125
  # Build cls_remap: for each model-emit index i,
126
  # cls_remap[i] = self.class_names.index(model_class_order[i])
127
+ # i.e. convert a model-side class id into the output class id that
128
+ # downstream code (BoundingBox.cls_id, the per-class threshold/bonus
129
+ # arrays) expects. The model-side order comes from the ONNX metadata
130
+ # when available, else falls back to the static _model_class_order.
131
  model_class_order = self._read_model_class_order()
132
  if model_class_order is None:
133
  model_class_order = list(self._model_class_order)
 
135
  else:
136
  print(f"cls order: from ONNX metadata {model_class_order}")
137
  self.cls_remap = np.array(
138
+ [self.class_names.index(n) for n in model_class_order],
139
+ dtype=np.int32,
140
  )
141
 
142
  for inp in self.session.get_inputs():
143
  print("INPUT:", inp.name, inp.shape, inp.type)
 
144
  for out in self.session.get_outputs():
145
  print("OUTPUT:", out.name, out.shape, out.type)
146
 
 
148
  self.output_names = [output.name for output in self.session.get_outputs()]
149
  self.input_shape = self.session.get_inputs()[0].shape
150
 
151
+ # weights.onnx is exported at 1280x1280 (Ultralytics imgsz metadata),
152
+ # static (dynamic=False). The default is only the fallback for when the
153
+ # ONNX input dims aren't fixed; the real value is read from the session.
 
 
 
154
  self.input_height = self._safe_dim(self.input_shape[2], default=1280)
155
  self.input_width = self._safe_dim(self.input_shape[3], default=1280)
156
 
 
 
 
 
 
 
 
 
 
 
157
  self.use_tta = False
158
+ self.use_tile_tta = False
159
+ # Soft-NMS (ported from carwash001): Gaussian score decay of overlapping
160
+ # boxes instead of hard removal. OFF by default to preserve the current
161
+ # deployed behaviour; flip on (and tune sigma) via tune_miner.py to see if
162
+ # it scores better useful where signs cluster (gantries, sign assemblies).
163
+ self.use_soft_nms = False
164
+ self.soft_nms_sigma = 0.5
165
+ self.soft_nms_score_thresh = 0.01
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
166
 
167
  print(f"✅ ONNX model loaded from: {model_path}")
168
  print(f"✅ ONNX providers: {self.session.get_providers()}")
169
  print(f"✅ ONNX input: name={self.input_name}, shape={self.input_shape}")
170
+ print(f"✅ ONNX input size: {self.input_width}x{self.input_height}, "
171
+ f"use_tta={self.use_tta}, use_tile_tta={self.use_tile_tta}")
172
+ print("per-class conf: " + ", ".join(
173
+ f"{n}={t:.3f}" for n, t in zip(
174
+ self.class_names, self._conf_thres_array.tolist()
175
+ )
176
+ ))
177
 
178
  self._warmup()
179
 
 
186
  except Exception as e:
187
  print(f"⚠️ warmup skipped: {e}")
188
 
189
+ def _read_model_class_order(self) -> "list[str] | None":
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
190
  """Read the model's class order from Ultralytics ONNX metadata.
191
+ Returns the class names ordered by model-emit index, or None when the
192
+ metadata is missing/unparsable or doesn't match `class_names` as a set
193
+ (in which case the static _model_class_order fallback is used)."""
 
 
194
  try:
195
  import ast
196
 
197
  meta = self.session.get_modelmeta().custom_metadata_map
198
+ names = ast.literal_eval(meta["names"]) # e.g. {0: 'road_sign'}
199
  if isinstance(names, dict):
200
  order = [str(names[i]) for i in sorted(names)]
201
  else:
 
211
  return None
212
  return order
213
 
214
+ def __repr__(self) -> str:
215
+ return (
216
+ f"ONNXRuntime(session={type(self.session).__name__}, "
217
+ f"providers={self.session.get_providers()})"
218
+ )
219
+
220
+ @staticmethod
221
+ def _safe_dim(value, default: int) -> int:
222
+ return value if isinstance(value, int) and value > 0 else default
223
+
224
  def _letterbox(
225
  self,
226
  image: ndarray,
227
  new_shape: tuple[int, int],
228
  color=(114, 114, 114),
229
  ) -> tuple[ndarray, float, tuple[float, float]]:
 
 
 
 
 
 
 
230
  h, w = image.shape[:2]
231
  new_w, new_h = new_shape
232
 
 
238
  interp = cv2.INTER_CUBIC if ratio > 1.0 else cv2.INTER_LINEAR
239
  image = cv2.resize(image, (resized_w, resized_h), interpolation=interp)
240
 
241
+ dw = (new_w - resized_w) / 2.0
242
+ dh = (new_h - resized_h) / 2.0
 
 
243
 
244
  left = int(round(dw - 0.1))
245
  right = int(round(dw + 0.1))
 
247
  bottom = int(round(dh + 0.1))
248
 
249
  padded = cv2.copyMakeBorder(
250
+ image, top, bottom, left, right,
251
+ borderType=cv2.BORDER_CONSTANT, value=color,
 
 
 
 
 
252
  )
253
  return padded, ratio, (dw, dh)
254
 
255
  def _preprocess(
256
  self, image: ndarray
257
  ) -> tuple[np.ndarray, float, tuple[float, float], tuple[int, int]]:
 
 
 
 
 
 
 
 
258
  orig_h, orig_w = image.shape[:2]
 
259
  img, ratio, pad = self._letterbox(
260
  image, (self.input_width, self.input_height)
261
  )
262
+ # Fused scale(1/255) + BGR->RGB swap + HWC->NCHW + contiguous float32 in
263
+ # one optimized OpenCV call (bit-identical to the cvtColor + astype/255 +
264
+ # transpose chain, but ~half the preprocess time).
265
+ blob = cv2.dnn.blobFromImage(img, scalefactor=1.0 / 255.0, swapRB=True)
266
+ return blob, ratio, pad, (orig_w, orig_h)
 
267
 
268
  @staticmethod
269
  def _clip_boxes(boxes: np.ndarray, image_size: tuple[int, int]) -> np.ndarray:
 
283
  out[:, 3] = boxes[:, 1] + boxes[:, 3] / 2.0
284
  return out
285
 
286
+ @staticmethod
287
+ def _hard_nms(
288
+ boxes: np.ndarray, scores: np.ndarray, iou_thresh: float
289
+ ) -> np.ndarray:
290
+ n = len(boxes)
291
+ if n == 0:
292
+ return np.array([], dtype=np.intp)
293
+ order = np.argsort(-scores)
294
+ keep: list[int] = []
295
+ while len(order) > 0:
296
+ i = int(order[0])
297
+ keep.append(i)
298
+ if len(order) == 1:
299
+ break
300
+ rest = order[1:]
301
+ xx1 = np.maximum(boxes[i, 0], boxes[rest, 0])
302
+ yy1 = np.maximum(boxes[i, 1], boxes[rest, 1])
303
+ xx2 = np.minimum(boxes[i, 2], boxes[rest, 2])
304
+ yy2 = np.minimum(boxes[i, 3], boxes[rest, 3])
305
+ inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
306
+ a_i = (max(0.0, boxes[i, 2] - boxes[i, 0]) *
307
+ max(0.0, boxes[i, 3] - boxes[i, 1]))
308
+ a_r = (np.maximum(0.0, boxes[rest, 2] - boxes[rest, 0]) *
309
+ np.maximum(0.0, boxes[rest, 3] - boxes[rest, 1]))
310
+ iou = inter / (a_i + a_r - inter + 1e-7)
311
+ order = rest[iou <= iou_thresh]
312
+ return np.array(keep, dtype=np.intp)
313
+
314
+ def _per_class_hard_nms(
315
+ self,
316
+ boxes: np.ndarray,
317
+ scores: np.ndarray,
318
+ cls_ids: np.ndarray,
319
+ iou_thresh: float,
320
+ ) -> np.ndarray:
321
+ if len(boxes) == 0:
322
+ return np.array([], dtype=np.intp)
323
+ all_keep: list[int] = []
324
+ for c in np.unique(cls_ids):
325
+ mask = cls_ids == c
326
+ indices = np.where(mask)[0]
327
+ keep = self._hard_nms(boxes[mask], scores[mask], iou_thresh)
328
+ all_keep.extend(indices[keep].tolist())
329
+ all_keep.sort()
330
+ return np.array(all_keep, dtype=np.intp)
331
+
332
  def _soft_nms(
333
  self,
334
  boxes: np.ndarray,
 
336
  sigma: float = 0.5,
337
  score_thresh: float = 0.01,
338
  ) -> tuple[np.ndarray, np.ndarray]:
339
+ """Soft-NMS: Gaussian decay of overlapping scores instead of hard removal.
340
+ Returns (kept_original_indices, updated_scores). (Ported from carwash001.)"""
 
 
341
  N = len(boxes)
342
  if N == 0:
343
  return np.array([], dtype=np.intp), np.array([], dtype=np.float32)
 
344
  boxes = boxes.astype(np.float32, copy=True)
345
  scores = scores.astype(np.float32, copy=True)
346
  order = np.arange(N)
 
347
  for i in range(N):
348
  max_pos = i + int(np.argmax(scores[i:]))
349
  boxes[[i, max_pos]] = boxes[[max_pos, i]]
350
  scores[[i, max_pos]] = scores[[max_pos, i]]
351
  order[[i, max_pos]] = order[[max_pos, i]]
 
352
  if i + 1 >= N:
353
  break
 
354
  xx1 = np.maximum(boxes[i, 0], boxes[i + 1:, 0])
355
  yy1 = np.maximum(boxes[i, 1], boxes[i + 1:, 1])
356
  xx2 = np.minimum(boxes[i, 2], boxes[i + 1:, 2])
357
  yy2 = np.minimum(boxes[i, 3], boxes[i + 1:, 3])
358
  inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
 
359
  area_i = max(0.0, float(
360
+ (boxes[i, 2] - boxes[i, 0]) * (boxes[i, 3] - boxes[i, 1])))
361
+ areas_j = (np.maximum(0.0, boxes[i + 1:, 2] - boxes[i + 1:, 0])
362
+ * np.maximum(0.0, boxes[i + 1:, 3] - boxes[i + 1:, 1]))
 
 
 
363
  iou = inter / (area_i + areas_j - inter + 1e-7)
364
  scores[i + 1:] *= np.exp(-(iou ** 2) / sigma)
 
365
  mask = scores > score_thresh
366
  return order[mask], scores[mask]
367
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
368
  def _per_class_soft_nms(
369
  self,
370
  boxes: np.ndarray,
 
373
  sigma: float = 0.5,
374
  score_thresh: float = 0.01,
375
  ) -> tuple[np.ndarray, np.ndarray]:
376
+ """Soft-NMS applied independently per class. Returns (kept_idx, updated_scores)."""
377
  if len(boxes) == 0:
378
  return np.array([], dtype=np.intp), np.array([], dtype=np.float32)
379
  all_keep: list[int] = []
380
  all_scores: list[float] = []
381
  for c in np.unique(cls_ids):
382
+ indices = np.where(cls_ids == c)[0]
383
+ keep, updated = self._soft_nms(boxes[indices], scores[indices],
384
+ sigma, score_thresh)
385
  for k, s in zip(keep, updated):
386
+ all_keep.append(int(indices[k])); all_scores.append(float(s))
 
387
  if not all_keep:
388
  return np.array([], dtype=np.intp), np.array([], dtype=np.float32)
389
  return np.array(all_keep, dtype=np.intp), np.array(all_scores, dtype=np.float32)
390
 
391
+ def _cross_class_dedup_op(
392
  self,
393
  boxes: np.ndarray,
394
  scores: np.ndarray,
395
  cls_ids: np.ndarray,
396
+ iou_thresh: float,
397
  ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
398
+ """Remove near-duplicate boxes across classes.
399
+ Order candidates by (score - per_class_threshold) margin, then by area;
400
+ keep the highest, suppress every other box with IoU > iou_thresh.
401
+ With a single road_sign class this is effectively a no-op, but the
402
+ method is kept so the pipeline stays compatible with the multi-class
403
+ miner template.
404
+ """
405
+ n = len(boxes)
406
+ if n <= 1:
407
  return boxes, scores, cls_ids
408
+ boxes = np.asarray(boxes, dtype=np.float32)
409
+ scores = np.asarray(scores, dtype=np.float32)
410
+ cls_ids = np.asarray(cls_ids, dtype=np.int32)
411
+ areas = (np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) *
412
+ np.maximum(0.0, boxes[:, 3] - boxes[:, 1]))
413
+ margins = scores - self._conf_thres_array[cls_ids]
414
+ order = np.lexsort((-areas, -margins))
415
+ suppressed = np.zeros(n, dtype=bool)
416
+ keep: list[int] = []
417
+ for i in order:
418
+ if suppressed[i]:
 
 
 
 
 
 
 
419
  continue
420
+ keep.append(int(i))
421
+ bi = boxes[i]
422
+ xx1 = np.maximum(bi[0], boxes[:, 0])
423
+ yy1 = np.maximum(bi[1], boxes[:, 1])
424
+ xx2 = np.minimum(bi[2], boxes[:, 2])
425
+ yy2 = np.minimum(bi[3], boxes[:, 3])
426
+ inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
427
+ a_i = max(1e-7, float((bi[2] - bi[0]) * (bi[3] - bi[1])))
428
+ iou = inter / (a_i + areas - inter + 1e-7)
429
+ dup = iou > iou_thresh
430
+ dup[i] = False
431
+ suppressed |= dup
432
+ keep_idx = np.array(keep, dtype=np.intp)
433
+ return boxes[keep_idx], scores[keep_idx], cls_ids[keep_idx]
434
 
435
  @staticmethod
436
  def _max_score_per_cluster(
 
442
  iou_thresh: float,
443
  ) -> np.ndarray:
444
  """For each kept (post-NMS) box, return the max score over the FULL
445
+ candidate set among same-class boxes with IoU >= iou_thresh.
446
+ Used after horizontal-flip TTA: a high-confidence flipped detection
447
+ can raise the score of the corresponding original detection.
 
 
448
  """
449
  n = len(post_boxes)
450
  if n == 0:
 
470
  ) -> np.ndarray:
471
  """Boolean keep-mask: score >= per-class threshold, with a per-class
472
  rescue -- if a class has zero boxes passing, admit its top-1 candidate
473
+ when its score >= (per-class threshold - per-class bonus)."""
 
474
  if len(scores) == 0:
475
  return np.zeros(0, dtype=bool)
476
  thr = self._conf_thres_array[cls_ids]
 
488
  keep[top] = True
489
  return keep
490
 
491
+ def _filter_sane_boxes(
492
  self,
493
  boxes: np.ndarray,
494
  scores: np.ndarray,
495
  cls_ids: np.ndarray,
496
+ orig_size: tuple[int, int],
497
  ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
498
+ """Drop tiny / degenerate / image-spanning / extreme-AR boxes (FP)."""
499
+ if len(boxes) == 0:
 
 
 
 
 
 
 
 
500
  return boxes, scores, cls_ids
501
+ orig_w, orig_h = orig_size
502
+ image_area = float(orig_w * orig_h)
503
+ keep = []
504
+ for i, box in enumerate(boxes):
505
+ x1, y1, x2, y2 = box.tolist()
506
+ bw = x2 - x1
507
+ bh = y2 - y1
508
+ if bw <= 0 or bh <= 0:
 
 
 
509
  continue
510
+ if bw < self.min_side or bh < self.min_side:
511
+ continue
512
+ area = bw * bh
513
+ if area < self.min_box_area:
514
+ continue
515
+ if area > 0.95 * image_area:
516
+ continue
517
+ ar = max(bw / max(bh, 1e-6), bh / max(bw, 1e-6))
518
+ if ar > self.max_aspect_ratio:
519
+ continue
520
+ keep.append(i)
521
+ if not keep:
522
+ return (
523
+ np.empty((0, 4), dtype=np.float32),
524
+ np.empty((0,), dtype=np.float32),
525
+ np.empty((0,), dtype=np.int32),
526
+ )
527
+ k = np.array(keep, dtype=np.intp)
528
+ return boxes[k], scores[k], cls_ids[k]
529
 
530
  def _per_view_pipeline(
531
  self,
 
533
  scores: np.ndarray,
534
  cls_ids: np.ndarray,
535
  ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
536
+ """Per-view post-processing pipeline: per-class NMS -> cap -> cross-class dedup."""
537
  if len(boxes) > 1:
538
+ if self.use_soft_nms:
539
+ keep, new_scores = self._per_class_soft_nms(
540
+ boxes, scores, cls_ids,
541
+ self.soft_nms_sigma, self.soft_nms_score_thresh)
542
+ boxes, scores, cls_ids = boxes[keep], new_scores, cls_ids[keep]
543
+ else:
544
+ keep = self._per_class_hard_nms(boxes, scores, cls_ids, self.iou_thres)
545
+ boxes, scores, cls_ids = boxes[keep], scores[keep], cls_ids[keep]
546
  if len(scores) > self.max_det:
547
  top = np.argsort(-scores)[: self.max_det]
548
  boxes, scores, cls_ids = boxes[top], scores[top], cls_ids[top]
 
552
  )
553
  return boxes, scores, cls_ids
554
 
555
+ @staticmethod
556
+ def _shrink_wh(coords: np.ndarray, sw: float, sh: float) -> np.ndarray:
557
+ """Shrink each xyxy box about its center: new_w = w/sw, new_h = h/sh."""
558
+ if len(coords) == 0:
559
+ return coords
560
+ coords = np.asarray(coords, dtype=np.float32).copy()
561
+ cx = (coords[:, 0] + coords[:, 2]) * 0.5
562
+ cy = (coords[:, 1] + coords[:, 3]) * 0.5
563
+ hw = (coords[:, 2] - coords[:, 0]) * (0.5 / sw)
564
+ hh = (coords[:, 3] - coords[:, 1]) * (0.5 / sh)
565
+ coords[:, 0] = cx - hw
566
+ coords[:, 1] = cy - hh
567
+ coords[:, 2] = cx + hw
568
+ coords[:, 3] = cy + hh
569
+ return coords
570
+
571
+ def _build_results(
572
+ self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray
573
+ ) -> list[BoundingBox]:
574
+ boxes = self._shrink_wh(boxes, self.box_shrink_w, self.box_shrink_h)
575
+ results: list[BoundingBox] = []
576
+ for box, conf, cls_id in zip(boxes, scores, cls_ids):
577
+ x1, y1, x2, y2 = box.tolist()
578
+ if x2 <= x1 or y2 <= y1:
579
+ continue
580
+ results.append(
581
+ BoundingBox(
582
+ x1=int(round(x1)),
583
+ y1=int(round(y1)),
584
+ x2=int(round(x2)),
585
+ y2=int(round(y2)),
586
+ cls_id=int(cls_id),
587
+ conf=float(conf),
588
+ )
589
+ )
590
+ return results
591
+
592
  def _decode_final_dets(
593
  self,
594
  preds: np.ndarray,
595
  ratio: float,
596
  pad: tuple[float, float],
597
  orig_size: tuple[int, int],
 
598
  ) -> list[BoundingBox]:
599
+ """Final-detection output path: rows shaped [x1, y1, x2, y2, conf, cls_id]."""
 
 
 
 
600
  if preds.ndim == 3 and preds.shape[0] == 1:
601
  preds = preds[0]
 
602
  if preds.ndim != 2 or preds.shape[1] < 6:
603
  raise ValueError(f"Unexpected ONNX final-det output shape: {preds.shape}")
604
 
 
607
  cls_ids = preds[:, 5].astype(np.int32)
608
  cls_ids = self.cls_remap[cls_ids]
609
 
 
610
  keep = self._conf_filter_mask(scores, cls_ids)
611
  boxes = boxes[keep]
612
  scores = scores[keep]
613
  cls_ids = cls_ids[keep]
 
614
  if len(boxes) == 0:
615
  return []
616
 
617
  pad_w, pad_h = pad
 
 
 
618
  boxes[:, [0, 2]] -= pad_w
619
  boxes[:, [1, 3]] -= pad_h
620
  boxes /= ratio
621
+ boxes = self._clip_boxes(boxes, orig_size)
622
 
 
623
  boxes, scores, cls_ids = self._filter_sane_boxes(
624
  boxes, scores, cls_ids, orig_size
625
  )
626
  if len(boxes) == 0:
627
  return []
628
 
629
+ boxes, scores, cls_ids = self._per_view_pipeline(boxes, scores, cls_ids)
630
+ return self._build_results(boxes, scores, cls_ids)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
631
 
632
  def _decode_raw_yolo(
633
  self,
 
636
  pad: tuple[float, float],
637
  orig_size: tuple[int, int],
638
  ) -> list[BoundingBox]:
639
+ """Fallback raw-YOLO output path: per-anchor class logits."""
640
+ if preds.ndim != 3 or preds.shape[0] != 1:
 
 
 
 
 
641
  raise ValueError(f"Unexpected raw ONNX output shape: {preds.shape}")
 
 
 
 
642
  preds = preds[0]
 
 
643
  if preds.shape[0] <= 16 and preds.shape[1] > preds.shape[0]:
644
  preds = preds.T
 
645
  if preds.ndim != 2 or preds.shape[1] < 5:
646
+ raise ValueError(f"Unexpected raw output shape: {preds.shape}")
647
 
648
  boxes_xywh = preds[:, :4].astype(np.float32)
649
  cls_part = preds[:, 4:].astype(np.float32)
 
650
  if cls_part.shape[1] == 1:
651
  scores = cls_part[:, 0]
652
  cls_ids = np.zeros(len(scores), dtype=np.int32)
 
655
  scores = cls_part[np.arange(len(cls_part)), cls_ids]
656
  cls_ids = self.cls_remap[cls_ids]
657
 
 
658
  keep = self._conf_filter_mask(scores, cls_ids)
659
  boxes_xywh = boxes_xywh[keep]
660
  scores = scores[keep]
661
  cls_ids = cls_ids[keep]
662
  if len(boxes_xywh) == 0:
663
  return []
 
664
  boxes = self._xywh_to_xyxy(boxes_xywh)
665
 
 
 
666
  pad_w, pad_h = pad
 
667
  boxes[:, [0, 2]] -= pad_w
668
  boxes[:, [1, 3]] -= pad_h
669
  boxes /= ratio
670
+ boxes = self._clip_boxes(boxes, orig_size)
671
 
672
  boxes, scores, cls_ids = self._filter_sane_boxes(
673
+ boxes, scores, cls_ids, orig_size
674
  )
675
  if len(boxes) == 0:
676
  return []
677
 
678
  boxes, scores, cls_ids = self._per_view_pipeline(boxes, scores, cls_ids)
679
+ return self._build_results(boxes, scores, cls_ids)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
680
 
681
  def _postprocess(
682
  self,
 
685
  pad: tuple[float, float],
686
  orig_size: tuple[int, int],
687
  ) -> list[BoundingBox]:
 
 
 
 
 
688
  if output.ndim == 2 and output.shape[1] >= 6:
689
  return self._decode_final_dets(output, ratio, pad, orig_size)
 
 
690
  if output.ndim == 3 and output.shape[0] == 1 and output.shape[2] == 6:
691
  return self._decode_final_dets(output, ratio, pad, orig_size)
 
 
692
  return self._decode_raw_yolo(output, ratio, pad, orig_size)
693
 
694
  def _predict_single(self, image: np.ndarray) -> list[BoundingBox]:
 
702
  raise ValueError(f"Invalid image shape={image.shape}")
703
  if image.shape[2] != 3:
704
  raise ValueError(f"Expected 3 channels, got shape={image.shape}")
 
705
  if image.dtype != np.uint8:
706
  image = image.astype(np.uint8)
707
 
708
  input_tensor, ratio, pad, orig_size = self._preprocess(image)
709
+ expected = (1, 3, self.input_height, self.input_width)
710
+ if input_tensor.shape != expected:
 
711
  raise ValueError(
712
+ f"Bad input tensor shape={input_tensor.shape}, expected={expected}"
713
  )
714
 
715
  outputs = self.session.run(self.output_names, {self.input_name: input_tensor})
716
+ return self._postprocess(outputs[0], ratio, pad, orig_size)
 
717
 
718
  def _predict_tta(self, image: np.ndarray) -> list[BoundingBox]:
719
  """Horizontal-flip TTA.
720
+ Strategy:
 
721
  1. Predict on original and on flipped image.
722
  2. Map flipped boxes back to original coordinates.
723
  3. Per-class hard NMS on the union.
724
+ 4. For each kept box, compute the max same-class score across the
725
+ FULL union (not just the post-NMS subset) -- this lets a high-
726
+ confidence flipped detection raise a borderline original one.
727
  5. Cross-class dedup to suppress same-physical-object multi-class.
728
  """
729
  boxes_orig = self._predict_single(image)
 
730
  flipped = cv2.flip(image, 1)
731
  boxes_flip = self._predict_single(flipped)
 
732
  w = image.shape[1]
733
  boxes_flip = [
734
  BoundingBox(
 
737
  )
738
  for b in boxes_flip
739
  ]
 
740
  all_boxes = boxes_orig + boxes_flip
741
+ if not all_boxes:
742
  return []
743
 
744
  coords = np.array(
 
754
  top = np.argsort(-scores[hard_keep])[: self.max_det]
755
  hard_keep = hard_keep[top]
756
 
 
 
757
  boosted = self._max_score_per_cluster(
758
  coords[hard_keep], cls_ids[hard_keep],
759
  coords, scores, cls_ids, self.iou_thres,
 
766
  kept_coords, boosted, kept_cls, self.cross_iou_thresh
767
  )
768
 
769
+ kept_coords = self._shrink_wh(
770
+ kept_coords, self.box_shrink_w, self.box_shrink_h
771
+ )
772
  return [
773
  BoundingBox(
774
+ x1=int(round(float(kept_coords[j, 0]))),
775
+ y1=int(round(float(kept_coords[j, 1]))),
776
+ x2=int(round(float(kept_coords[j, 2]))),
777
+ y2=int(round(float(kept_coords[j, 3]))),
778
  cls_id=int(kept_cls[j]),
779
  conf=float(boosted[j]),
780
  )
781
  for j in range(len(kept_coords))
782
  ]
783
 
784
+ def _predict_tiles(self, image: np.ndarray) -> list[BoundingBox]:
785
+ """Tile-based TTA for high-resolution images.
786
+ Splits the source image into two overlapping horizontal tiles, runs
787
+ single-pass inference on each at native scale, and translates boxes
788
+ back to the global frame. Useful when source width >> model input
789
+ width because letterboxing otherwise discards effective resolution
790
+ that small / distant signs depend on.
791
+ Returns an empty list if the image isn't wide enough to benefit; the
792
+ caller falls back to the regular pipeline in that case.
793
+ """
794
+ h, w = image.shape[:2]
795
+ if w < int(self.input_width * self.tile_trigger_ratio):
796
+ return []
797
+
798
+ overlap = int(w * self.tile_overlap_ratio)
799
+ mid = w // 2
800
+ x_left_end = min(w, mid + overlap // 2)
801
+ x_right_start = max(0, mid - overlap // 2)
802
+
803
+ left = image[:, :x_left_end]
804
+ right = image[:, x_right_start:]
805
+
806
+ boxes_left = self._predict_single(left)
807
+ boxes_right = self._predict_single(right)
808
+
809
+ shifted_right = [
810
+ BoundingBox(
811
+ x1=b.x1 + x_right_start,
812
+ y1=b.y1,
813
+ x2=b.x2 + x_right_start,
814
+ y2=b.y2,
815
+ cls_id=b.cls_id,
816
+ conf=b.conf,
817
+ )
818
+ for b in boxes_right
819
+ ]
820
+ return boxes_left + shifted_right
821
+
822
+ def _merge_views(
823
+ self,
824
+ view_boxes: list[list[BoundingBox]],
825
+ image_size: tuple[int, int],
826
+ ) -> list[BoundingBox]:
827
+ """Merge boxes from multiple views (single / hflip / tiles).
828
+ Same logic as `_predict_tta`'s tail: per-class hard NMS to dedupe,
829
+ then for each kept box take the max same-class score across the full
830
+ candidate union — a high-confidence detection in any view boosts
831
+ borderline matches in others.
832
+ """
833
+ all_boxes: list[BoundingBox] = []
834
+ for vb in view_boxes:
835
+ all_boxes.extend(vb)
836
+ if not all_boxes:
837
+ return []
838
+
839
+ coords = np.array(
840
+ [[b.x1, b.y1, b.x2, b.y2] for b in all_boxes], dtype=np.float32
841
+ )
842
+ scores = np.array([b.conf for b in all_boxes], dtype=np.float32)
843
+ cls_ids = np.array([b.cls_id for b in all_boxes], dtype=np.int32)
844
+
845
+ coords = self._clip_boxes(coords, image_size)
846
+
847
+ hard_keep = self._per_class_hard_nms(coords, scores, cls_ids, self.iou_thres)
848
+ if len(hard_keep) == 0:
849
+ return []
850
+ if len(hard_keep) > self.max_det:
851
+ top = np.argsort(-scores[hard_keep])[: self.max_det]
852
+ hard_keep = hard_keep[top]
853
+
854
+ boosted = self._max_score_per_cluster(
855
+ coords[hard_keep], cls_ids[hard_keep],
856
+ coords, scores, cls_ids, self.iou_thres,
857
+ )
858
+
859
+ kept_coords = coords[hard_keep]
860
+ kept_cls = cls_ids[hard_keep]
861
+ if len(kept_coords) > 1:
862
+ kept_coords, boosted, kept_cls = self._cross_class_dedup_op(
863
+ kept_coords, boosted, kept_cls, self.cross_iou_thresh
864
+ )
865
+
866
+ kept_coords = self._shrink_wh(
867
+ kept_coords, self.box_shrink_w, self.box_shrink_h
868
+ )
869
+ return [
870
+ BoundingBox(
871
+ x1=int(round(float(kept_coords[j, 0]))),
872
+ y1=int(round(float(kept_coords[j, 1]))),
873
+ x2=int(round(float(kept_coords[j, 2]))),
874
+ y2=int(round(float(kept_coords[j, 3]))),
875
+ cls_id=int(kept_cls[j]),
876
+ conf=float(boosted[j]),
877
+ )
878
+ for j in range(len(kept_coords))
879
+ ]
880
+
881
+ def _predict_full(self, image: np.ndarray) -> list[BoundingBox]:
882
+ """Top-level per-frame prediction with all enabled augmentations.
883
+ - `use_tta=True`: original + horizontal flip
884
+ - `use_tile_tta=True` AND image wide enough: two overlapping tiles
885
+ All views are merged via per-class NMS + cluster-max score boost.
886
+ """
887
+ if not self.use_tta and not self.use_tile_tta:
888
+ return self._predict_single(image)
889
+
890
+ views: list[list[BoundingBox]] = []
891
+ if self.use_tta:
892
+ views.append(self._predict_single(image))
893
+ flipped = cv2.flip(image, 1)
894
+ w = image.shape[1]
895
+ flipped_dets = self._predict_single(flipped)
896
+ views.append([
897
+ BoundingBox(
898
+ x1=w - b.x2, y1=b.y1, x2=w - b.x1, y2=b.y2,
899
+ cls_id=b.cls_id, conf=b.conf,
900
+ )
901
+ for b in flipped_dets
902
+ ])
903
+ else:
904
+ views.append(self._predict_single(image))
905
+
906
+ if self.use_tile_tta:
907
+ tile_boxes = self._predict_tiles(image)
908
+ if tile_boxes:
909
+ views.append(tile_boxes)
910
+
911
+ h, w = image.shape[:2]
912
+ return self._merge_views(views, (w, h))
913
+
914
  def predict_batch(
915
  self,
916
  batch_images: list[ndarray],
 
918
  n_keypoints: int,
919
  ) -> list[TVFrameResult]:
920
  results: list[TVFrameResult] = []
 
921
  for frame_number_in_batch, image in enumerate(batch_images):
922
  try:
923
+ boxes = self._predict_full(image)
 
 
 
924
  except Exception as e:
925
+ print(
926
+ f"⚠️ Inference failed for frame "
927
+ f"{offset + frame_number_in_batch}: {e}"
928
+ )
929
  boxes = []
 
930
  results.append(
931
  TVFrameResult(
932
  frame_id=offset + frame_number_in_batch,
 
934
  keypoints=[(0, 0) for _ in range(max(0, int(n_keypoints)))],
935
  )
936
  )
937
+ return results
 
weights.onnx CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:756f81aa467d482bb6508d172cc20c34fbb4cb7f15eeabb5538b858a1e561c94
3
- size 9824686
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5eb4543ec44fe7dd6a743e36b687ddd08985e54fdd358426b118bec1c4972ad7
3
+ size 9805054