from pathlib import Path import math import cv2 import numpy as np import onnxruntime as ort from numpy import ndarray from pydantic import BaseModel class BoundingBox(BaseModel): x1: int y1: int x2: int y2: int cls_id: int conf: float class TVFrameResult(BaseModel): frame_id: int boxes: list[BoundingBox] keypoints: list[tuple[int, int]] class Miner: class_names = ['fire', 'smoke', 'fire extinguisher'] _model_class_order = ['fire', 'smoke', 'fire extinguisher'] iou_thres = 0.5 max_det = 30 _conf_thres_array = np.array([0.10, 0.10, 0.20], dtype=np.float32) _bonus_array = np.array([0.05, 0.05, 0.05], dtype=np.float32) min_box_area = 100 min_side = 8 max_aspect_ratio = 8.0 smoke_merge_overlap = 0.35 fire_suppress_overlap = 0.9 ext_scale = 0.95 fire_expand = 1.01 fire_color_filter_max_conf = 0.45 color_filter_min_saturation = 0.06 def __init__(self, path_hf_repo: Path) -> None: model_path = path_hf_repo / 'weights.onnx' print('ORT version:', ort.__version__) try: ort.preload_dlls() print('✅ onnxruntime.preload_dlls() success') except Exception as e: print(f'⚠️ preload_dlls failed: {e}') print('ORT available providers BEFORE session:', ort.get_available_providers()) sess_options = ort.SessionOptions() sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL sess_options.intra_op_num_threads = 2 sess_options.inter_op_num_threads = 1 sess_options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL try: self.session = ort.InferenceSession(str(model_path), sess_options=sess_options, providers=['CUDAExecutionProvider', 'CPUExecutionProvider']) print('✅ Created ORT session with preferred CUDA provider list') except Exception as e: print(f'⚠️ CUDA session creation failed, falling back to CPU: {e}') self.session = ort.InferenceSession(str(model_path), sess_options=sess_options, providers=['CPUExecutionProvider']) print('ORT session providers:', self.session.get_providers()) model_class_order = self._read_model_class_order() if model_class_order is None: model_class_order = list(self._model_class_order) print(f'cls order: no usable ONNX metadata, FALLBACK {model_class_order}') else: print(f'cls order: from ONNX metadata {model_class_order}') self.cls_remap = np.array([self.class_names.index(n) for n in model_class_order], dtype=np.int32) for inp in self.session.get_inputs(): print('INPUT:', inp.name, inp.shape, inp.type) for out in self.session.get_outputs(): print('OUTPUT:', out.name, out.shape, out.type) self.input_name = self.session.get_inputs()[0].name self.output_names = [output.name for output in self.session.get_outputs()] self.input_shape = self.session.get_inputs()[0].shape self.input_height = self._safe_dim(self.input_shape[2], default=1280) self.input_width = self._safe_dim(self.input_shape[3], default=1280) print(f'✅ ONNX model loaded from: {model_path}') print(f'✅ ONNX providers: {self.session.get_providers()}') print(f'✅ ONNX input: name={self.input_name}, shape={self.input_shape}') print('per-class conf: ' + ', '.join((f'{n}={t:.3f}' for n, t in zip(self.class_names, self._conf_thres_array.tolist())))) self._warmup() def _warmup(self, iters: int=3) -> None: try: dummy = np.zeros((720, 1280, 3), dtype=np.uint8) for _ in range(max(1, iters)): self.predict_batch(batch_images=[dummy], offset=0, n_keypoints=0) print(f'✅ warmup: {iters} dummy predict_batch call(s) done') except Exception as e: print(f'⚠️ warmup skipped: {e}') def __repr__(self) -> str: return f'ONNXRuntime(session={type(self.session).__name__}, providers={self.session.get_providers()})' @staticmethod def _safe_dim(value, default: int) -> int: return value if isinstance(value, int) and value > 0 else default def _read_model_class_order(self) -> list[str] | None: try: import ast meta = self.session.get_modelmeta().custom_metadata_map names = ast.literal_eval(meta['names']) if isinstance(names, dict): order = [str(names[i]) for i in sorted(names)] else: order = [str(n) for n in names] except Exception as e: print(f'cls order: could not read ONNX names metadata ({e})') return None if sorted(order) != sorted(self.class_names): print(f'cls order: ONNX names {order} do not match expected classes {self.class_names}; ignoring metadata') return None return order def _letterbox(self, image: ndarray, new_shape: tuple[int, int], color=(114, 114, 114)) -> tuple[ndarray, float, tuple[float, float]]: h, w = image.shape[:2] new_w, new_h = new_shape ratio = min(new_w / w, new_h / h) resized_w = int(round(w * ratio)) resized_h = int(round(h * ratio)) if (resized_w, resized_h) != (w, h): interp = cv2.INTER_CUBIC if ratio > 1.0 else cv2.INTER_LINEAR image = cv2.resize(image, (resized_w, resized_h), interpolation=interp) dw = (new_w - resized_w) / 2.0 dh = (new_h - resized_h) / 2.0 left = int(round(dw - 0.1)) right = int(round(dw + 0.1)) top = int(round(dh - 0.1)) bottom = int(round(dh + 0.1)) padded = cv2.copyMakeBorder(image, top, bottom, left, right, borderType=cv2.BORDER_CONSTANT, value=color) return (padded, ratio, (dw, dh)) def _preprocess(self, image: ndarray) -> tuple[np.ndarray, float, tuple[float, float], tuple[int, int]]: orig_h, orig_w = image.shape[:2] img, ratio, pad = self._letterbox(image, (self.input_width, self.input_height)) blob = cv2.dnn.blobFromImage(img, scalefactor=1.0 / 255.0, swapRB=True) return (blob, ratio, pad, (orig_w, orig_h)) @staticmethod def _clip_boxes(boxes: np.ndarray, image_size: tuple[int, int]) -> np.ndarray: w, h = image_size boxes[:, 0] = np.clip(boxes[:, 0], 0, w - 1) boxes[:, 1] = np.clip(boxes[:, 1], 0, h - 1) boxes[:, 2] = np.clip(boxes[:, 2], 0, w - 1) boxes[:, 3] = np.clip(boxes[:, 3], 0, h - 1) return boxes @staticmethod def _hard_nms(boxes: np.ndarray, scores: np.ndarray, iou_thresh: float) -> np.ndarray: n = len(boxes) if n == 0: return np.array([], dtype=np.intp) order = np.argsort(-scores) keep: list[int] = [] while len(order) > 0: i = int(order[0]) keep.append(i) if len(order) == 1: break rest = order[1:] xx1 = np.maximum(boxes[i, 0], boxes[rest, 0]) yy1 = np.maximum(boxes[i, 1], boxes[rest, 1]) xx2 = np.minimum(boxes[i, 2], boxes[rest, 2]) yy2 = np.minimum(boxes[i, 3], boxes[rest, 3]) inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1) a_i = max(0.0, boxes[i, 2] - boxes[i, 0]) * max(0.0, boxes[i, 3] - boxes[i, 1]) a_r = np.maximum(0.0, boxes[rest, 2] - boxes[rest, 0]) * np.maximum(0.0, boxes[rest, 3] - boxes[rest, 1]) iou = inter / (a_i + a_r - inter + 1e-07) order = rest[iou <= iou_thresh] return np.array(keep, dtype=np.intp) def _per_class_hard_nms(self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, iou_thresh: float) -> np.ndarray: if len(boxes) == 0: return np.array([], dtype=np.intp) all_keep: list[int] = [] for c in np.unique(cls_ids): mask = cls_ids == c indices = np.where(mask)[0] keep = self._hard_nms(boxes[mask], scores[mask], iou_thresh) all_keep.extend(indices[keep].tolist()) all_keep.sort() return np.array(all_keep, dtype=np.intp) def _order_by_margin(self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]: boxes = np.asarray(boxes, dtype=np.float32) scores = np.asarray(scores, dtype=np.float32) cls_ids = np.asarray(cls_ids, dtype=np.int32) areas = np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) * np.maximum(0.0, boxes[:, 3] - boxes[:, 1]) margins = scores - self._conf_thres_array[cls_ids] order = np.lexsort((-areas, -margins)) return (boxes[order], scores[order], cls_ids[order]) def _merge_smoke_boxes(self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]: target_cls = self.class_names.index('smoke') overlap = self.smoke_merge_overlap idx = np.where(cls_ids == target_cls)[0] if len(idx) <= 1: return (boxes, scores, cls_ids) sb = boxes[idx].astype(np.float32).tolist() ss = scores[idx].astype(np.float32).tolist() merged_any = True while merged_any and len(sb) > 1: merged_any = False for i in range(len(sb)): for j in range(i + 1, len(sb)): a, b = (sb[i], sb[j]) ix1 = max(a[0], b[0]) iy1 = max(a[1], b[1]) ix2 = min(a[2], b[2]) iy2 = min(a[3], b[3]) inter = max(0.0, ix2 - ix1) * max(0.0, iy2 - iy1) area_a = max(0.0, a[2] - a[0]) * max(0.0, a[3] - a[1]) area_b = max(0.0, b[2] - b[0]) * max(0.0, b[3] - b[1]) smaller = min(area_a, area_b) if inter / (smaller + 1e-07) >= overlap: sb[i] = [min(a[0], b[0]), min(a[1], b[1]), max(a[2], b[2]), max(a[3], b[3])] ss[i] = max(ss[i], ss[j]) del sb[j] del ss[j] merged_any = True break if merged_any: break other = cls_ids != target_cls new_boxes = np.concatenate([boxes[other].astype(np.float32), np.array(sb, dtype=np.float32).reshape(-1, 4)]) new_scores = np.concatenate([scores[other].astype(np.float32), np.array(ss, dtype=np.float32)]) new_cls = np.concatenate([cls_ids[other].astype(np.int32), np.full(len(sb), target_cls, dtype=np.int32)]) return (new_boxes, new_scores, new_cls) def _suppress_contained_fire(self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]: target_cls = self.class_names.index('fire') overlap = self.fire_suppress_overlap idx = np.where(cls_ids == target_cls)[0] if len(idx) <= 1: return (boxes, scores, cls_ids) order = idx[np.argsort(-scores[idx])] remove: set[int] = set() for a in range(len(order)): i = int(order[a]) if i in remove: continue bi = boxes[i] area_i = max(1e-07, float((bi[2] - bi[0]) * (bi[3] - bi[1]))) for b in range(a + 1, len(order)): j = int(order[b]) if j in remove: continue bj = boxes[j] ix1 = max(bi[0], bj[0]) iy1 = max(bi[1], bj[1]) ix2 = min(bi[2], bj[2]) iy2 = min(bi[3], bj[3]) inter = max(0.0, ix2 - ix1) * max(0.0, iy2 - iy1) if inter <= 0.0: continue area_j = max(1e-07, float((bj[2] - bj[0]) * (bj[3] - bj[1]))) if inter / (min(area_i, area_j) + 1e-07) >= overlap: remove.add(j) if not remove: return (boxes, scores, cls_ids) keep = np.array([k not in remove for k in range(len(boxes))], dtype=bool) return (boxes[keep], scores[keep], cls_ids[keep]) def _merge_same_class_boxes(self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]: boxes, scores, cls_ids = self._merge_smoke_boxes(boxes, scores, cls_ids) return self._suppress_contained_fire(boxes, scores, cls_ids) def _conf_filter_mask(self, scores: np.ndarray, cls_ids: np.ndarray) -> np.ndarray: if len(scores) == 0: return np.zeros(0, dtype=bool) thr = self._conf_thres_array[cls_ids] keep = scores >= thr for c in np.unique(cls_ids): b = float(self._bonus_array[c]) if b <= 0.0: continue cm = cls_ids == c if keep[cm].any(): continue idx = np.where(cm)[0] top = int(idx[int(np.argmax(scores[idx]))]) if scores[top] >= self._conf_thres_array[c] - b: keep[top] = True return keep def _filter_sane_boxes(self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, orig_size: tuple[int, int]) -> tuple[np.ndarray, np.ndarray, np.ndarray]: if len(boxes) == 0: return (boxes, scores, cls_ids) orig_w, orig_h = orig_size image_area = float(orig_w * orig_h) keep = [] for i, box in enumerate(boxes): x1, y1, x2, y2 = box.tolist() bw = x2 - x1 bh = y2 - y1 if bw <= 0 or bh <= 0: continue if bw < self.min_side or bh < self.min_side: continue area = bw * bh if area < self.min_box_area: continue if area > 0.95 * image_area: continue ar = max(bw / max(bh, 1e-06), bh / max(bw, 1e-06)) if ar > self.max_aspect_ratio: continue keep.append(i) if not keep: return (np.empty((0, 4), dtype=np.float32), np.empty((0,), dtype=np.float32), np.empty((0,), dtype=np.int32)) k = np.array(keep, dtype=np.intp) return (boxes[k], scores[k], cls_ids[k]) def _per_view_pipeline(self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]: if len(boxes) > 1: keep = self._per_class_hard_nms(boxes, scores, cls_ids, self.iou_thres) boxes, scores, cls_ids = (boxes[keep], scores[keep], cls_ids[keep]) if len(scores) > self.max_det: top = np.argsort(-scores)[:self.max_det] boxes, scores, cls_ids = (boxes[top], scores[top], cls_ids[top]) if len(boxes) > 1: boxes, scores, cls_ids = self._order_by_margin(boxes, scores, cls_ids) if len(boxes) > 1: boxes, scores, cls_ids = self._merge_same_class_boxes(boxes, scores, cls_ids) return (boxes, scores, cls_ids) @staticmethod def _roi_for_box(image: np.ndarray, box: BoundingBox) -> np.ndarray | None: h, w = image.shape[:2] x1 = max(0, int(math.floor(box.x1))) y1 = max(0, int(math.floor(box.y1))) x2 = min(w, int(math.ceil(box.x2))) y2 = min(h, int(math.ceil(box.y2))) if x2 <= x1 or y2 <= y1: return None roi = image[y1:y2, x1:x2] return roi if roi.size else None def _roi_is_near_grayscale(self, roi: np.ndarray) -> bool: mx = roi.max(axis=2).astype(np.float32) mn = roi.min(axis=2).astype(np.float32) sat = (mx - mn) / (mx + 1e-06) return float(sat.mean()) < self.color_filter_min_saturation @staticmethod def _passes_fire_color(roi: np.ndarray) -> bool: blue = roi[:, :, 0].astype(np.float32) green = roi[:, :, 1].astype(np.float32) red = roi[:, :, 2].astype(np.float32) mean_r = float(np.mean(red)) max_rgb = float(max(np.max(red), np.max(green), np.max(blue))) bright_frac = float(np.mean(np.max(roi, axis=2) >= 150)) if max_rgb >= 200.0 and bright_frac >= 0.01: return True warm = (red > green + 10.0) & (red > blue + 10.0) warm_frac = float(np.mean(warm)) r_minus_g = mean_r - float(np.mean(green)) if warm_frac >= 0.05 and (max_rgb >= 120.0 or mean_r >= 120.0 or warm_frac >= 0.15): return True if bright_frac >= 0.12 and r_minus_g >= 2.0: return True return False def _filter_low_conf_by_color(self, image: np.ndarray, results: list[BoundingBox]) -> list[BoundingBox]: if not results: return results cls_fire = self.class_names.index('fire') out: list[BoundingBox] = [] for box in results: if box.cls_id != cls_fire or box.conf > self.fire_color_filter_max_conf: out.append(box) continue roi = self._roi_for_box(image, box) if roi is None or self._roi_is_near_grayscale(roi): out.append(box) continue if not self._passes_fire_color(roi): continue out.append(box) return out @staticmethod def _build_results(boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray) -> list[BoundingBox]: results: list[BoundingBox] = [] for box, conf, cls_id in zip(boxes, scores, cls_ids): x1, y1, x2, y2 = box.tolist() if x2 <= x1 or y2 <= y1: continue results.append(BoundingBox(x1=int(math.floor(x1)), y1=int(math.floor(y1)), x2=int(math.ceil(x2)), y2=int(math.ceil(y2)), cls_id=int(cls_id), conf=float(conf))) return results @staticmethod def _empty_raw() -> tuple[np.ndarray, np.ndarray, np.ndarray]: return (np.empty((0, 4), dtype=np.float32), np.empty((0,), dtype=np.float32), np.empty((0,), dtype=np.int32)) def _renms_when_raw_smoke(self, finals: list[BoundingBox], raw_boxes: np.ndarray, raw_cls: np.ndarray) -> list[BoundingBox]: if not finals or len(raw_boxes) == 0 or len(finals) <= 1: return finals if not np.any(raw_cls == self.class_names.index('smoke')): return finals boxes = np.array([[b.x1, b.y1, b.x2, b.y2] for b in finals], dtype=np.float32) scores = np.array([b.conf for b in finals], dtype=np.float32) cls_ids = np.array([b.cls_id for b in finals], dtype=np.int32) keep = self._per_class_hard_nms(boxes, scores, cls_ids, self.iou_thres) return [finals[int(i)] for i in keep] def _rescale_class_boxes(self, finals: list[BoundingBox], orig_size: tuple[int, int]) -> list[BoundingBox]: if not finals: return finals img_w, img_h = orig_size fire_id = self.class_names.index('fire') ext_id = self.class_names.index('fire extinguisher') out: list[BoundingBox] = [] for b in finals: x1, y1, x2, y2 = (float(b.x1), float(b.y1), float(b.x2), float(b.y2)) w = max(0.0, x2 - x1) h = max(0.0, y2 - y1) if w <= 0.0 or h <= 0.0: continue if b.cls_id == ext_id: scale = float(self.ext_scale) nw, nh = (w * scale, h * scale) cx = 0.5 * (x1 + x2) nx1 = cx - 0.5 * nw nx2 = cx + 0.5 * nw ny2 = y2 ny1 = ny2 - nh elif b.cls_id == fire_id: scale = float(self.fire_expand) nw, nh = (w * scale, h * scale) cx = 0.5 * (x1 + x2) cy = 0.5 * (y1 + y2) nx1 = cx - 0.5 * nw nx2 = cx + 0.5 * nw ny1 = cy - 0.5 * nh ny2 = cy + 0.5 * nh else: out.append(b) continue nx1 = max(0.0, min(float(img_w), nx1)) nx2 = max(0.0, min(float(img_w), nx2)) ny1 = max(0.0, min(float(img_h), ny1)) ny2 = max(0.0, min(float(img_h), ny2)) if nx2 <= nx1 or ny2 <= ny1: continue out.append(BoundingBox(x1=int(math.floor(nx1)), y1=int(math.floor(ny1)), x2=int(math.ceil(nx2)), y2=int(math.ceil(ny2)), cls_id=b.cls_id, conf=b.conf)) return out def _apply_extra_post(self, finals: list[BoundingBox], raw_boxes: np.ndarray, raw_cls: np.ndarray, orig_size: tuple[int, int]) -> list[BoundingBox]: finals = self._renms_when_raw_smoke(finals, raw_boxes, raw_cls) return self._rescale_class_boxes(finals, orig_size) def _postprocess(self, preds: np.ndarray, ratio: float, pad: tuple[float, float], orig_size: tuple[int, int]) -> tuple[list[BoundingBox], tuple[np.ndarray, np.ndarray, np.ndarray]]: if preds.ndim == 3 and preds.shape[0] == 1: preds = preds[0] if preds.ndim != 2 or preds.shape[1] < 6: raise ValueError(f'Unexpected ONNX final-det output shape: {preds.shape}') boxes = preds[:, :4].astype(np.float32) scores = preds[:, 4].astype(np.float32) cls_ids = self.cls_remap[preds[:, 5].astype(np.int32)] keep = self._conf_filter_mask(scores, cls_ids) boxes = boxes[keep] scores = scores[keep] cls_ids = cls_ids[keep] if len(boxes) == 0: return ([], self._empty_raw()) pad_w, pad_h = pad boxes[:, [0, 2]] -= pad_w boxes[:, [1, 3]] -= pad_h boxes /= ratio boxes = self._clip_boxes(boxes, orig_size) raw = (boxes, scores, cls_ids) boxes, scores, cls_ids = self._filter_sane_boxes(boxes, scores, cls_ids, orig_size) if len(boxes) == 0: return ([], raw) boxes, scores, cls_ids = self._per_view_pipeline(boxes, scores, cls_ids) return (self._build_results(boxes, scores, cls_ids), raw) def _predict_single(self, image: np.ndarray) -> tuple[list[BoundingBox], tuple[np.ndarray, np.ndarray, np.ndarray]]: if image is None: raise ValueError('Input image is None') if not isinstance(image, np.ndarray): raise TypeError(f'Input is not numpy array: {type(image)}') if image.ndim != 3: raise ValueError(f'Expected HWC image, got shape={image.shape}') if image.shape[0] <= 0 or image.shape[1] <= 0: raise ValueError(f'Invalid image shape={image.shape}') if image.shape[2] != 3: raise ValueError(f'Expected 3 channels, got shape={image.shape}') if image.dtype != np.uint8: image = image.astype(np.uint8) input_tensor, ratio, pad, orig_size = self._preprocess(image) expected = (1, 3, self.input_height, self.input_width) if input_tensor.shape != expected: raise ValueError(f'Bad input tensor shape={input_tensor.shape}, expected={expected}') outputs = self.session.run(self.output_names, {self.input_name: input_tensor}) return self._postprocess(outputs[0], ratio, pad, orig_size) def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]: results: list[TVFrameResult] = [] for frame_number_in_batch, image in enumerate(batch_images): try: boxes, raw = self._predict_single(image) if isinstance(image, np.ndarray) and image.ndim == 3: boxes = self._filter_low_conf_by_color(image, boxes) boxes = self._apply_extra_post(boxes, raw[0], raw[2], (image.shape[1], image.shape[0])) else: boxes = self._apply_extra_post(boxes, raw[0], raw[2], (0, 0)) except Exception as e: print(f'⚠️ Inference failed for frame {offset + frame_number_in_batch}: {e}') boxes = [] results.append(TVFrameResult(frame_id=offset + frame_number_in_batch, boxes=boxes, keypoints=[(0, 0) for _ in range(max(0, int(n_keypoints)))])) return results