firenet-v8d-hybrid / miner.py
maple-matrix's picture
scorevision: push artifact
ff0002e verified
Raw
History Blame Contribute Delete
129 kB
from pathlib import Path
import math
import cv2
import numpy as np
import onnxruntime as ort
from numpy import ndarray
from pydantic import BaseModel
class BoundingBox(BaseModel):
x1: int
y1: int
x2: int
y2: int
cls_id: int
conf: float
class TVFrameResult(BaseModel):
frame_id: int
boxes: list[BoundingBox]
keypoints: list[tuple[int, int]]
class Miner:
class_names = ["fire", "smoke", "fire extinguisher"]
_model_class_order = ["fire", "fire extinguisher", "smoke"]
_parent_split_class_names = ("fire", "fire extinguisher")
conf_thresh = 0.000005
isolated_conf_thresh = 0.008
same_class_iou_thresh = 0.6
edge_conf_thresh = 0.005
edge_tol = 1.0
overlap_low_conf_thresh = 0.003
overlap_same_class_iou_thresh = 0.5
group_iou_thresh = 0.7
flip_low_conf_thresh = 0.2
fire_ext_same_object_overlap_thresh = 0.7
fire_ext_group_involve_thresh = 0.9
smoke_involve_thresh = 0.7
smoke_prefer_larger_conf_thresh = 0.3
smoke_cluster_merge_involve_thresh = 0.3
smoke_cluster_merge_min_size = 2
smoke_cluster_merge_early_min_count = 3
small_group_min_count = 2
small_group_low_conf_thresh = 0.02
fire_parent_min_small_count = 2
fire_parent_involve_thresh = 0.7
fire_split_inner_involve_thresh = 0.5
fire_split_max_inner_area_ratio = 0.5
fire_split_cluster_iou_thresh = 0.5
fire_split_min_clusters = 2
fire_split_min_inner_conf = 0.08
fire_split_peak_member_min_conf = 0.05
fire_split_peak_dist_ratio = 0.10
fire_split_peak_dist_min_px = 12.0
fire_split_peak_dist_max_px = 48.0
fire_split_centroid_preserve_px = 12.0
fire_split_conf_boost = 0.2
fire_split_parent_blob_conf_thresh = 0.5
fire_min_conf = 0.165
smoke_min_conf = 0.07
fire_ext_min_conf = 0.037
fire_ext_color_filter_max_conf = 0.20
fire_ext_min_red_dom_frac = 0.03
fire_ext_color_min_mean_r = 50.0
fire_ext_min_r_minus_g = 0.0
smoke_anchor_min_smoke_conf = 0.63
smoke_anchor_base_height_frac = 0.35
smoke_anchor_crop_width_pad_ratio = 0.15
smoke_anchor_crop_down_pad_px = 24.0
smoke_anchor_corroborate_involve_thresh = 0.5
smoke_anchor_fire_floor_conf = 0.165
smoke_anchor_min_fire_width_px = 10.0
smoke_anchor_min_fire_height_px = 10.0
smoke_anchor_strong_smoke_conf = 0.7
smoke_anchor_corroborated_min_fire_width_px = 4.0
smoke_anchor_corroborated_min_fire_height_px = 4.0
smoke_anchor_max_fire_area_ratio = 0.25
smoke_anchor_corroborate_area_ratio_max = 4.0
smoke_anchor_max_fires_per_smoke = 2
smoke_anchor_max_crop_fires = 2
smoke_anchor_max_probe_smokes = 2
smoke_anchor_separate_fire_involve_thresh = 0.5
smoke_anchor_max_probe_fire_width_px = 52.0
smoke_anchor_max_probe_fire_height_px = 48.0
smoke_anchor_max_probe_fire_area_ratio = 0.04
smoke_anchor_max_probe_crop_width_ratio = 0.25
smoke_anchor_max_probe_crop_height_ratio = 0.60
smoke_anchor_max_probe_fire_bottom_dist_frac = 0.10
smoke_anchor_probe_rescue_min_conf = 0.000005
smoke_anchor_corroborated_probe_min_conf = 0.00005
smoke_anchor_corroborated_refine_pad_px = 3.0
smoke_anchor_expanded_width_ratio = 1.25
ext_probe_min_cluster_count = 2
ext_probe_cluster_centroid_dist_px = 28.0
ext_probe_min_cluster_best_conf = 0.003
ext_probe_crop_pad_ratio = 0.85
ext_probe_min_crop_size_px = 200.0
ext_probe_crop_rescue_min_conf = 0.5
ext_probe_crop_rescue_min_width_px = 6.0
ext_probe_rescue_min_conf = 0.000005
ext_probe_min_width_px = 10.0
ext_probe_min_height_px = 10.0
ext_probe_max_width_px = 120.0
ext_probe_max_height_px = 160.0
ext_probe_fire_exclude_involve_thresh = 0.5
ext_probe_near_fire_gap_px = 24.0
ext_probe_duplicate_involve_thresh = 0.5
ext_probe_max_per_frame = 2
ext_probe_edge_tol = 1.0
ext_probe_singleton_min_conf = 0.00005
ext_probe_singleton_max_area = 500.0
ext_probe_singleton_max_width_px = 20.0
ext_probe_singleton_max_height_px = 35.0
ext_probe_singleton_min_edge_margin_px = 12.0
smoke_anchor_probe_min_mean_r = 65.0
smoke_anchor_probe_min_max_rgb = 80.0
smoke_anchor_probe_min_warm_frac = 0.05
smoke_anchor_probe_warm_min_max_rgb = 120.0
smoke_anchor_probe_warm_min_mean_r = 120.0
smoke_anchor_probe_strong_warm_frac = 0.15
smoke_anchor_probe_min_bright_frac = 0.12
smoke_anchor_probe_bright_thresh = 150
smoke_anchor_probe_min_hot_max_rgb = 180.0
smoke_anchor_probe_min_r_minus_g = 2.0
smoke_anchor_probe_sparse_hot_min_max_rgb = 200.0
smoke_anchor_probe_sparse_hot_min_bright_frac = 0.01
smoke_expand_min_seed_conf = 0.5
smoke_expand_crop_width_pad_ratio = 0.45
smoke_expand_crop_up_pad_ratio = 0.15
smoke_expand_crop_down_pad_px = 32.0
smoke_expand_corroborate_involve_thresh = 0.3
smoke_expand_vertical_band_pad_ratio = 0.10
smoke_expand_band_max_horizontal_gap_ratio = 0.25
smoke_expand_upward_diagonal_max_gap_ratio = 0.45
smoke_expand_min_probe_smoke_conf = 0.10
smoke_expand_min_tta_smoke_conf = 0.12
smoke_expand_min_tta_extend_conf = 0.15
smoke_expand_tta_view_iou_thresh = 0.5
smoke_expand_wide_seed_max_width_ratio = 0.25
smoke_expand_upper_plume_max_y_ratio = 0.72
smoke_expand_wide_crop_x_start_ratio = 0.05
smoke_expand_wide_crop_x_end_ratio = 0.95
smoke_expand_min_crop_width_ratio = 0.60
smoke_expand_border_strip_ratio = 0.15
smoke_expand_max_color_dist = 45.0
smoke_expand_max_frame_ratio = 0.20
smoke_expand_min_extend_probe_conf = 0.15
smoke_expand_min_bidi_crop_conf = 0.40
smoke_expand_min_upward_tta_conf = 0.012
smoke_expand_min_upward_tta_involve_conf = 0.0002
smoke_expand_min_upward_crop_conf = 0.012
smoke_expand_max_seeds = 1
smoke_expand_skip_min_conf = 0.85
smoke_expand_skip_min_width_ratio = 0.15
smoke_expand_skip_max_y1_ratio = 0.04
def __init__(self, path_hf_repo: Path) -> None:
model_path = path_hf_repo / "weights.onnx"
self.group_object_counts: list[int] = []
self.group_max_confidences: list[float] = []
self.group_avg_confidences: list[float] = []
self.cls_remap = np.array(
[self.class_names.index(n) for n in self._model_class_order],
dtype=np.int32,
)
print("ORT version:", ort.__version__)
try:
ort.preload_dlls()
print("✅ onnxruntime.preload_dlls() success")
except Exception as e:
print(f"⚠️ preload_dlls failed: {e}")
print("ORT available providers BEFORE session:", ort.get_available_providers())
sess_options = ort.SessionOptions()
sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
try:
self.session = ort.InferenceSession(
str(model_path),
sess_options=sess_options,
providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
)
print("✅ Created ORT session with preferred CUDA provider list")
except Exception as e:
print(f"⚠️ CUDA session creation failed, falling back to CPU: {e}")
self.session = ort.InferenceSession(
str(model_path),
sess_options=sess_options,
providers=["CPUExecutionProvider"],
)
print("ORT session providers:", self.session.get_providers())
for inp in self.session.get_inputs():
print("INPUT:", inp.name, inp.shape, inp.type)
for out in self.session.get_outputs():
print("OUTPUT:", out.name, out.shape, out.type)
self.input_name = self.session.get_inputs()[0].name
self.output_names = [output.name for output in self.session.get_outputs()]
self.input_shape = self.session.get_inputs()[0].shape
self.input_height = self._safe_dim(self.input_shape[2], default=1280)
self.input_width = self._safe_dim(self.input_shape[3], default=1280)
self.use_tta = True
print(f"✅ ONNX model loaded from: {model_path}")
print(f"✅ ONNX providers: {self.session.get_providers()}")
print(f"✅ ONNX input: name={self.input_name}, shape={self.input_shape}")
def __repr__(self) -> str:
return (
f"ONNXRuntime(session={type(self.session).__name__}, "
f"providers={self.session.get_providers()})"
)
@staticmethod
def _safe_dim(value, default: int) -> int:
return value if isinstance(value, int) and value > 0 else default
def _letterbox(
self,
image: ndarray,
new_shape: tuple[int, int],
color=(114, 114, 114),
) -> tuple[ndarray, float, tuple[float, float]]:
h, w = image.shape[:2]
new_w, new_h = new_shape
ratio = min(new_w / w, new_h / h)
resized_w = int(round(w * ratio))
resized_h = int(round(h * ratio))
if (resized_w, resized_h) != (w, h):
interp = cv2.INTER_CUBIC if ratio > 1.0 else cv2.INTER_LINEAR
image = cv2.resize(image, (resized_w, resized_h), interpolation=interp)
dw = (new_w - resized_w) / 2.0
dh = (new_h - resized_h) / 2.0
left = int(round(dw - 0.1))
right = int(round(dw + 0.1))
top = int(round(dh - 0.1))
bottom = int(round(dh + 0.1))
padded = cv2.copyMakeBorder(
image, top, bottom, left, right,
borderType=cv2.BORDER_CONSTANT, value=color,
)
return padded, ratio, (dw, dh)
def _preprocess(
self, image: ndarray
) -> tuple[np.ndarray, float, tuple[float, float], tuple[int, int]]:
orig_h, orig_w = image.shape[:2]
img, ratio, pad = self._letterbox(
image, (self.input_width, self.input_height)
)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = img.astype(np.float32) / 255.0
img = np.transpose(img, (2, 0, 1))[None, ...]
img = np.ascontiguousarray(img, dtype=np.float32)
return img, ratio, pad, (orig_w, orig_h)
@staticmethod
def _clip_boxes(boxes: np.ndarray, image_size: tuple[int, int]) -> np.ndarray:
w, h = image_size
boxes[:, 0] = np.clip(boxes[:, 0], 0, w - 1)
boxes[:, 1] = np.clip(boxes[:, 1], 0, h - 1)
boxes[:, 2] = np.clip(boxes[:, 2], 0, w - 1)
boxes[:, 3] = np.clip(boxes[:, 3], 0, h - 1)
return boxes
@staticmethod
def _build_results(
boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray
) -> list[BoundingBox]:
results: list[BoundingBox] = []
for box, conf, cls_id in zip(boxes, scores, cls_ids):
x1, y1, x2, y2 = box.tolist()
if x2 <= x1 or y2 <= y1:
continue
results.append(
BoundingBox(
x1=int(math.floor(x1)),
y1=int(math.floor(y1)),
x2=int(math.ceil(x2)),
y2=int(math.ceil(y2)),
cls_id=int(cls_id),
conf=float(conf),
)
)
return results
def _map_boxes_to_orig(
self,
boxes: np.ndarray,
ratio: float,
pad: tuple[float, float],
orig_size: tuple[int, int],
) -> np.ndarray:
pad_w, pad_h = pad
boxes = boxes.copy()
boxes[:, [0, 2]] -= pad_w
boxes[:, [1, 3]] -= pad_h
boxes /= ratio
return self._clip_boxes(boxes, orig_size)
@staticmethod
def _compute_iou_matrix(boxes: np.ndarray) -> np.ndarray:
n = len(boxes)
if n == 0:
return np.empty((0, 0), dtype=np.float32)
areas = (np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) *
np.maximum(0.0, boxes[:, 3] - boxes[:, 1]))
iou = np.zeros((n, n), dtype=np.float32)
for i in range(n):
bi = boxes[i]
xx1 = np.maximum(bi[0], boxes[i + 1:, 0])
yy1 = np.maximum(bi[1], boxes[i + 1:, 1])
xx2 = np.minimum(bi[2], boxes[i + 1:, 2])
yy2 = np.minimum(bi[3], boxes[i + 1:, 3])
inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
a_i = max(1e-7, float((bi[2] - bi[0]) * (bi[3] - bi[1])))
iou[i, i + 1:] = inter / (a_i + areas[i + 1:] - inter + 1e-7)
iou = iou + iou.T
np.fill_diagonal(iou, 1.0)
return iou
def _group_overlapping_detections(
self,
boxes: np.ndarray,
scores: np.ndarray,
cls_ids: np.ndarray,
iou_thresh: float | None = None,
view_ids: np.ndarray | None = None,
record_stats: bool = True,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
if iou_thresh is None:
iou_thresh = self.group_iou_thresh
n = len(boxes)
if record_stats:
self.group_object_counts = []
self.group_max_confidences = []
self.group_avg_confidences = []
if n == 0:
return boxes, scores, cls_ids
boxes = np.asarray(boxes, dtype=np.float32)
scores = np.asarray(scores, dtype=np.float32)
cls_ids = np.asarray(cls_ids, dtype=np.int32)
if view_ids is not None:
view_ids = np.asarray(view_ids, dtype=np.int32)
parent = list(range(n))
def find(x: int) -> int:
while parent[x] != x:
parent[x] = parent[parent[x]]
x = parent[x]
return x
def union(a: int, b: int) -> None:
ra, rb = find(a), find(b)
if ra != rb:
parent[rb] = ra
iou = self._compute_iou_matrix(boxes)
for i in range(n):
for j in range(i + 1, n):
if cls_ids[i] != cls_ids[j]:
continue
if view_ids is not None and view_ids[i] != view_ids[j]:
continue
if iou[i, j] > iou_thresh:
union(i, j)
groups: dict[int, list[int]] = {}
for i in range(n):
groups.setdefault(find(i), []).append(i)
kept_groups: list[tuple[int, int, float, float]] = []
for members in groups.values():
member_scores = scores[members]
count = len(members)
max_conf = float(np.max(member_scores))
avg_conf = float(np.mean(member_scores))
best = members[int(np.argmax(member_scores))]
kept_groups.append((best, count, max_conf, avg_conf))
kept_groups.sort(key=lambda item: item[0])
keep = np.array([item[0] for item in kept_groups], dtype=np.intp)
if record_stats:
self.group_object_counts = [item[1] for item in kept_groups]
self.group_max_confidences = [item[2] for item in kept_groups]
self.group_avg_confidences = [item[3] for item in kept_groups]
return boxes[keep], scores[keep], cls_ids[keep]
def _remove_isolated_low_conf_same_class(
self,
boxes: np.ndarray,
scores: np.ndarray,
cls_ids: np.ndarray,
view_ids: np.ndarray | None = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray] | tuple[
np.ndarray, np.ndarray, np.ndarray, np.ndarray
]:
n = len(boxes)
if n == 0:
if view_ids is None:
return boxes, scores, cls_ids
return boxes, scores, cls_ids, view_ids
if n == 1:
if scores[0] < self.isolated_conf_thresh:
empty_boxes = np.empty((0, 4), dtype=np.float32)
empty_scores = np.empty((0,), dtype=np.float32)
empty_cls = np.empty((0,), dtype=np.int32)
if view_ids is None:
return empty_boxes, empty_scores, empty_cls
return empty_boxes, empty_scores, empty_cls, np.empty((0,), dtype=np.int32)
if view_ids is None:
return boxes, scores, cls_ids
return boxes, scores, cls_ids, view_ids
boxes = np.asarray(boxes, dtype=np.float32)
scores = np.asarray(scores, dtype=np.float32)
cls_ids = np.asarray(cls_ids, dtype=np.int32)
areas = (np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) *
np.maximum(0.0, boxes[:, 3] - boxes[:, 1]))
keep = np.ones(n, dtype=bool)
low_conf = scores < self.isolated_conf_thresh
if not np.any(low_conf):
if view_ids is None:
return boxes, scores, cls_ids
return boxes, scores, cls_ids, view_ids
for i in np.where(low_conf)[0]:
bi = boxes[i]
xx1 = np.maximum(bi[0], boxes[:, 0])
yy1 = np.maximum(bi[1], boxes[:, 1])
xx2 = np.minimum(bi[2], boxes[:, 2])
yy2 = np.minimum(bi[3], boxes[:, 3])
inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
a_i = max(1e-7, float((bi[2] - bi[0]) * (bi[3] - bi[1])))
iou = inter / (a_i + areas - inter + 1e-7)
same_class = cls_ids == cls_ids[i]
other = np.arange(n) != i
if not np.any((iou > self.same_class_iou_thresh) & same_class & other):
keep[i] = False
if view_ids is None:
return boxes[keep], scores[keep], cls_ids[keep]
return boxes[keep], scores[keep], cls_ids[keep], view_ids[keep]
def _remove_edge_low_conf(
self,
boxes: np.ndarray,
scores: np.ndarray,
cls_ids: np.ndarray,
orig_size: tuple[int, int],
view_ids: np.ndarray | None = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray] | tuple[
np.ndarray, np.ndarray, np.ndarray, np.ndarray
]:
if len(boxes) == 0:
if view_ids is None:
return boxes, scores, cls_ids
return boxes, scores, cls_ids, view_ids
orig_w, orig_h = orig_size
tol = self.edge_tol
on_edge = (
(boxes[:, 0] <= tol)
| (boxes[:, 1] <= tol)
| (boxes[:, 2] >= orig_w - 1 - tol)
| (boxes[:, 3] >= orig_h - 1 - tol)
)
keep = ~(on_edge & (scores < self.edge_conf_thresh))
if view_ids is None:
return boxes[keep], scores[keep], cls_ids[keep]
return boxes[keep], scores[keep], cls_ids[keep], view_ids[keep]
def _remove_overlapping_low_conf_same_class(
self,
boxes: np.ndarray,
scores: np.ndarray,
cls_ids: np.ndarray,
view_ids: np.ndarray | None = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray] | tuple[
np.ndarray, np.ndarray, np.ndarray, np.ndarray
]:
n = len(boxes)
if n <= 1:
if view_ids is None:
return boxes, scores, cls_ids
return boxes, scores, cls_ids, view_ids
boxes = np.asarray(boxes, dtype=np.float32)
scores = np.asarray(scores, dtype=np.float32)
cls_ids = np.asarray(cls_ids, dtype=np.int32)
areas = (np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) *
np.maximum(0.0, boxes[:, 3] - boxes[:, 1]))
keep = np.ones(n, dtype=bool)
low_conf = scores < self.overlap_low_conf_thresh
if not np.any(low_conf):
if view_ids is None:
return boxes, scores, cls_ids
return boxes, scores, cls_ids, view_ids
for i in np.where(low_conf)[0]:
bi = boxes[i]
xx1 = np.maximum(bi[0], boxes[:, 0])
yy1 = np.maximum(bi[1], boxes[:, 1])
xx2 = np.minimum(bi[2], boxes[:, 2])
yy2 = np.minimum(bi[3], boxes[:, 3])
inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
a_i = max(1e-7, float((bi[2] - bi[0]) * (bi[3] - bi[1])))
iou = inter / (a_i + areas - inter + 1e-7)
same_class = cls_ids == cls_ids[i]
other = np.arange(n) != i
if np.any((iou >= self.overlap_same_class_iou_thresh) & same_class & other):
keep[i] = False
if view_ids is None:
return boxes[keep], scores[keep], cls_ids[keep]
return boxes[keep], scores[keep], cls_ids[keep], view_ids[keep]
def _decode_preds_to_arrays(
self,
preds: np.ndarray,
ratio: float,
pad: tuple[float, float],
orig_size: tuple[int, int],
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
if preds.ndim == 3 and preds.shape[0] == 1:
preds = preds[0]
if preds.ndim != 2 or preds.shape[1] < 6:
raise ValueError(f"Unexpected ONNX final-det output shape: {preds.shape}")
boxes = preds[:, :4].astype(np.float32)
scores = preds[:, 4].astype(np.float32)
cls_ids = self.cls_remap[preds[:, 5].astype(np.int32)]
keep = scores >= self.conf_thresh
boxes = boxes[keep]
scores = scores[keep]
cls_ids = cls_ids[keep]
if len(boxes) == 0:
return (
np.empty((0, 4), dtype=np.float32),
np.empty((0,), dtype=np.float32),
np.empty((0,), dtype=np.int32),
)
boxes = self._map_boxes_to_orig(boxes, ratio, pad, orig_size)
valid = (boxes[:, 2] > boxes[:, 0]) & (boxes[:, 3] > boxes[:, 1])
return boxes[valid], scores[valid], cls_ids[valid]
@staticmethod
def _map_flipped_boxes_arrays(
boxes: np.ndarray,
orig_size: tuple[int, int],
flip_code: int,
) -> np.ndarray:
orig_w, _orig_h = orig_size
out = boxes.copy()
if flip_code == 1:
x1 = orig_w - out[:, 2]
x2 = orig_w - out[:, 0]
out[:, 0] = x1
out[:, 2] = x2
else:
raise ValueError(f"Unsupported flip_code: {flip_code}")
return out
def _remove_unconfirmed_single_view_low_conf(
self,
boxes: np.ndarray,
scores: np.ndarray,
cls_ids: np.ndarray,
view_ids: np.ndarray,
conf_thresh: float | None = None,
match_iou_thresh: float | None = None,
protect_mask: np.ndarray | None = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
if conf_thresh is None:
conf_thresh = self.flip_low_conf_thresh
if match_iou_thresh is None:
match_iou_thresh = self.overlap_same_class_iou_thresh
n = len(boxes)
if n == 0:
return boxes, scores, cls_ids, view_ids
low_conf = scores < conf_thresh
if not np.any(low_conf):
return boxes, scores, cls_ids, view_ids
iou = self._compute_iou_matrix(boxes)
keep = np.ones(n, dtype=bool)
for i in np.where(low_conf)[0]:
if protect_mask is not None and protect_mask[i]:
continue
other = np.arange(n) != i
other_view = view_ids != view_ids[i]
same_class = cls_ids == cls_ids[i]
corroborated = np.any(
other & other_view & same_class & (iou[i] >= match_iou_thresh)
)
if not corroborated:
keep[i] = False
return boxes[keep], scores[keep], cls_ids[keep], view_ids[keep]
@staticmethod
def _boxes_involve_each_other(
boxes: np.ndarray, i: int, j: int, overlap_thresh: float
) -> bool:
bi, bj = boxes[i], boxes[j]
xx1 = max(float(bi[0]), float(bj[0]))
yy1 = max(float(bi[1]), float(bj[1]))
xx2 = min(float(bi[2]), float(bj[2]))
yy2 = min(float(bi[3]), float(bj[3]))
inter = max(0.0, xx2 - xx1) * max(0.0, yy2 - yy1)
if inter <= 0.0:
return False
area_i = max(1e-7, float((bi[2] - bi[0]) * (bi[3] - bi[1])))
area_j = max(1e-7, float((bj[2] - bj[0]) * (bj[3] - bj[1])))
return (inter / min(area_i, area_j)) >= overlap_thresh
@staticmethod
def _box_area(box: np.ndarray) -> float:
return max(1e-7, float((box[2] - box[0]) * (box[3] - box[1])))
@staticmethod
def _box_intersection_area(box_a: np.ndarray, box_b: np.ndarray) -> float:
xx1 = max(float(box_a[0]), float(box_b[0]))
yy1 = max(float(box_a[1]), float(box_b[1]))
xx2 = min(float(box_a[2]), float(box_b[2]))
yy2 = min(float(box_a[3]), float(box_b[3]))
return max(0.0, xx2 - xx1) * max(0.0, yy2 - yy1)
@staticmethod
def _larger_involves_smaller(
boxes: np.ndarray, big_idx: int, small_idx: int, involve_thresh: float
) -> bool:
inter = Miner._box_intersection_area(boxes[big_idx], boxes[small_idx])
if inter <= 0.0:
return False
area_small = Miner._box_area(boxes[small_idx])
return (inter / area_small) >= involve_thresh
@staticmethod
def _either_box_involves_other(
boxes: np.ndarray, i: int, j: int, involve_thresh: float
) -> bool:
inter = Miner._box_intersection_area(boxes[i], boxes[j])
if inter <= 0.0:
return False
area_i = Miner._box_area(boxes[i])
area_j = Miner._box_area(boxes[j])
return (inter / area_i >= involve_thresh) or (inter / area_j >= involve_thresh)
@staticmethod
def _boxes_involve_each_other_arrays(
box_a: np.ndarray, box_b: np.ndarray, involve_thresh: float
) -> bool:
inter = Miner._box_intersection_area(box_a, box_b)
if inter <= 0.0:
return False
area_a = Miner._box_area(box_a)
area_b = Miner._box_area(box_b)
return (inter / area_a >= involve_thresh) or (inter / area_b >= involve_thresh)
def _resolve_smoke_overlap(
self,
boxes: np.ndarray,
scores: np.ndarray,
cls_ids: np.ndarray,
involve_thresh: float | None = None,
prefer_larger_conf_thresh: float | None = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
if involve_thresh is None:
involve_thresh = self.smoke_involve_thresh
if prefer_larger_conf_thresh is None:
prefer_larger_conf_thresh = self.smoke_prefer_larger_conf_thresh
n = len(boxes)
if n <= 1:
return boxes, scores, cls_ids
cls_smoke = self.class_names.index("smoke")
keep = np.ones(n, dtype=bool)
for i in range(n):
if not keep[i] or int(cls_ids[i]) != cls_smoke:
continue
for j in range(i + 1, n):
if not keep[j] or int(cls_ids[j]) != cls_smoke:
continue
area_i = self._box_area(boxes[i])
area_j = self._box_area(boxes[j])
if area_i == area_j:
big_idx, small_idx = (i, j) if scores[i] >= scores[j] else (j, i)
elif area_i > area_j:
big_idx, small_idx = i, j
else:
big_idx, small_idx = j, i
if not self._larger_involves_smaller(
boxes, big_idx, small_idx, involve_thresh
):
continue
big_conf = float(scores[big_idx])
small_conf = float(scores[small_idx])
if (
big_conf > prefer_larger_conf_thresh
and small_conf > prefer_larger_conf_thresh
):
keep[small_idx] = False
elif big_conf >= small_conf:
keep[small_idx] = False
else:
keep[big_idx] = False
if big_idx == i:
break
return boxes[keep], scores[keep], cls_ids[keep]
def _merge_overlapping_smoke_clusters(
self,
boxes: np.ndarray,
scores: np.ndarray,
cls_ids: np.ndarray,
involve_thresh: float | None = None,
min_cluster_size: int | None = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
if involve_thresh is None:
involve_thresh = self.smoke_cluster_merge_involve_thresh
if min_cluster_size is None:
min_cluster_size = self.smoke_cluster_merge_min_size
n = len(boxes)
if n == 0:
return boxes, scores, cls_ids
cls_smoke = self.class_names.index("smoke")
smoke_indices = [
i for i in range(n) if int(cls_ids[i]) == cls_smoke
]
if len(smoke_indices) < min_cluster_size:
return boxes, scores, cls_ids
n_smoke = len(smoke_indices)
parent = list(range(n_smoke))
def find(x: int) -> int:
while parent[x] != x:
parent[x] = parent[parent[x]]
x = parent[x]
return x
def union(a: int, b: int) -> None:
ra, rb = find(a), find(b)
if ra != rb:
parent[rb] = ra
for ai in range(n_smoke):
for bi in range(ai + 1, n_smoke):
idx_i = smoke_indices[ai]
idx_j = smoke_indices[bi]
if self._either_box_involves_other(
boxes, idx_i, idx_j, involve_thresh
):
union(ai, bi)
clusters: dict[int, list[int]] = {}
for ai in range(n_smoke):
root = find(ai)
clusters.setdefault(root, []).append(smoke_indices[ai])
remove_indices: set[int] = set()
merged_boxes: list[np.ndarray] = []
merged_scores: list[float] = []
for cluster in clusters.values():
if len(cluster) < min_cluster_size:
continue
remove_indices.update(cluster)
union_box = boxes[cluster[0]].copy()
max_conf = float(scores[cluster[0]])
for idx in cluster[1:]:
union_box[0] = min(union_box[0], boxes[idx][0])
union_box[1] = min(union_box[1], boxes[idx][1])
union_box[2] = max(union_box[2], boxes[idx][2])
union_box[3] = max(union_box[3], boxes[idx][3])
max_conf = max(max_conf, float(scores[idx]))
merged_boxes.append(union_box)
merged_scores.append(max_conf)
if not remove_indices:
return boxes, scores, cls_ids
keep = np.ones(n, dtype=bool)
for idx in remove_indices:
keep[idx] = False
out_boxes = list(boxes[keep])
out_scores = [float(s) for s in scores[keep]]
out_cls = [int(c) for c in cls_ids[keep]]
out_boxes.extend(merged_boxes)
out_scores.extend(merged_scores)
out_cls.extend([cls_smoke] * len(merged_boxes))
return (
np.asarray(out_boxes, dtype=np.float32),
np.asarray(out_scores, dtype=np.float32),
np.asarray(out_cls, dtype=np.int32),
)
def _remove_class_min_conf(
self,
boxes: np.ndarray,
scores: np.ndarray,
cls_ids: np.ndarray,
fire_min_conf: float | None = None,
smoke_min_conf: float | None = None,
fire_ext_min_conf: float | None = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
if fire_min_conf is None:
fire_min_conf = self.fire_min_conf
if smoke_min_conf is None:
smoke_min_conf = self.smoke_min_conf
if fire_ext_min_conf is None:
fire_ext_min_conf = self.fire_ext_min_conf
n = len(boxes)
if n == 0:
return boxes, scores, cls_ids
cls_fire = self.class_names.index("fire")
cls_smoke = self.class_names.index("smoke")
cls_fire_ext = self.class_names.index("fire extinguisher")
keep = np.ones(n, dtype=bool)
for i in range(n):
cls_id = int(cls_ids[i])
if cls_id == cls_fire and float(scores[i]) < fire_min_conf:
keep[i] = False
elif cls_id == cls_smoke and float(scores[i]) < smoke_min_conf:
keep[i] = False
elif cls_id == cls_fire_ext and float(scores[i]) < fire_ext_min_conf:
keep[i] = False
return boxes[keep], scores[keep], cls_ids[keep]
def _filter_results_min_conf(
self, results: list[BoundingBox]
) -> list[BoundingBox]:
if not results:
return results
cls_fire = self.class_names.index("fire")
cls_smoke = self.class_names.index("smoke")
cls_fire_ext = self.class_names.index("fire extinguisher")
filtered: list[BoundingBox] = []
for box in results:
if box.cls_id == cls_fire and box.conf < self.fire_min_conf:
continue
if box.cls_id == cls_smoke and box.conf < self.smoke_min_conf:
continue
if box.cls_id == cls_fire_ext and box.conf < self.fire_ext_min_conf:
continue
filtered.append(box)
return filtered
def _resolve_fire_and_ext_same_class_overlap(
self,
boxes: np.ndarray,
scores: np.ndarray,
cls_ids: np.ndarray,
overlap_thresh: float | None = None,
group_involve_thresh: float | None = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
if overlap_thresh is None:
overlap_thresh = self.fire_ext_same_object_overlap_thresh
if group_involve_thresh is None:
group_involve_thresh = self.fire_ext_group_involve_thresh
n = len(boxes)
if n <= 1:
return boxes, scores, cls_ids
cls_fire = self.class_names.index("fire")
dedupe_classes = {
cls_fire,
self.class_names.index("fire extinguisher"),
}
keep = np.ones(n, dtype=bool)
for i in range(n):
if not keep[i] or int(cls_ids[i]) not in dedupe_classes:
continue
for j in range(i + 1, n):
if not keep[j] or cls_ids[i] != cls_ids[j]:
continue
should_dedupe = self._either_box_involves_other(
boxes, i, j, group_involve_thresh
)
if not should_dedupe:
if not self._boxes_involve_each_other(
boxes, i, j, overlap_thresh
):
continue
if int(cls_ids[i]) == cls_fire:
cx_i, cy_i = self._box_centroid(boxes[i])
cx_j, cy_j = self._box_centroid(boxes[j])
dx = cx_i - cx_j
dy = cy_i - cy_j
if (
dx * dx + dy * dy
> self.fire_split_centroid_preserve_px
* self.fire_split_centroid_preserve_px
):
continue
should_dedupe = True
if not should_dedupe:
continue
if scores[i] >= scores[j]:
keep[j] = False
else:
keep[i] = False
break
return boxes[keep], scores[keep], cls_ids[keep]
def _parent_split_cls_ids(self) -> list[int]:
return [self.class_names.index(name) for name in self._parent_split_class_names]
def _remove_spanning_parent_fire(
self,
boxes: np.ndarray,
scores: np.ndarray,
cls_ids: np.ndarray,
view_ids: np.ndarray | None = None,
involve_thresh: float | None = None,
min_small_count: int | None = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray] | tuple[
np.ndarray, np.ndarray, np.ndarray, np.ndarray
]:
if involve_thresh is None:
involve_thresh = self.fire_parent_involve_thresh
if min_small_count is None:
min_small_count = self.fire_parent_min_small_count
n = len(boxes)
if n <= 1:
if view_ids is None:
return boxes, scores, cls_ids
return boxes, scores, cls_ids, view_ids
keep = np.ones(n, dtype=bool)
for target_cls in self._parent_split_cls_ids():
class_indices = [i for i in range(n) if int(cls_ids[i]) == target_cls]
if len(class_indices) <= 1:
continue
for a_idx in class_indices:
area_a = self._box_area(boxes[a_idx])
small_indices = [
j
for j in class_indices
if j != a_idx
and self._box_area(boxes[j]) < area_a
and self._larger_involves_smaller(
boxes, a_idx, j, involve_thresh
)
]
if len(small_indices) < min_small_count:
continue
smalls_are_separate = True
for i_pos in range(len(small_indices)):
for j_pos in range(i_pos + 1, len(small_indices)):
if self._boxes_involve_each_other(
boxes,
small_indices[i_pos],
small_indices[j_pos],
involve_thresh,
):
smalls_are_separate = False
break
if not smalls_are_separate:
break
if smalls_are_separate:
keep[a_idx] = False
if view_ids is None:
return boxes[keep], scores[keep], cls_ids[keep]
return boxes[keep], scores[keep], cls_ids[keep], view_ids[keep]
@staticmethod
def _box_centroid(box: np.ndarray) -> tuple[float, float]:
return ((box[0] + box[2]) / 2.0, (box[1] + box[3]) / 2.0)
@staticmethod
def _centroid_inside_box(
boxes: np.ndarray, parent_idx: int, child_idx: int
) -> bool:
parent = boxes[parent_idx]
cx, cy = Miner._box_centroid(boxes[child_idx])
return parent[0] <= cx <= parent[2] and parent[1] <= cy <= parent[3]
@staticmethod
def _mostly_inside_parent(
boxes: np.ndarray,
parent_idx: int,
child_idx: int,
involve_thresh: float,
) -> bool:
inter = Miner._box_intersection_area(boxes[parent_idx], boxes[child_idx])
if inter <= 0.0:
return False
area_child = Miner._box_area(boxes[child_idx])
if (inter / area_child) >= involve_thresh:
return True
child = boxes[child_idx]
parent = boxes[parent_idx]
cx = (child[0] + child[2]) / 2.0
cy = (child[1] + child[3]) / 2.0
return (
parent[0] <= cx <= parent[2]
and parent[1] <= cy <= parent[3]
)
def _cluster_indices_by_box_iou(
self,
boxes: np.ndarray,
indices: list[int],
iou_thresh: float,
) -> list[list[int]]:
if not indices:
return []
if len(indices) == 1:
return [indices]
parent_map = {idx: idx for idx in indices}
def find(x: int) -> int:
while parent_map[x] != x:
parent_map[x] = parent_map[parent_map[x]]
x = parent_map[x]
return x
def union(a: int, b: int) -> None:
ra, rb = find(a), find(b)
if ra != rb:
parent_map[rb] = ra
iou = self._compute_iou_matrix(boxes)
for i_pos in range(len(indices)):
for j_pos in range(i_pos + 1, len(indices)):
a_idx = indices[i_pos]
b_idx = indices[j_pos]
if iou[a_idx, b_idx] > iou_thresh:
union(a_idx, b_idx)
groups: dict[int, list[int]] = {}
for idx in indices:
groups.setdefault(find(idx), []).append(idx)
return list(groups.values())
def _cluster_indices_by_centroid_distance(
self,
boxes: np.ndarray,
indices: list[int],
max_dist: float,
) -> list[list[int]]:
if not indices:
return []
if len(indices) == 1:
return [indices]
parent_map = {idx: idx for idx in indices}
def find(x: int) -> int:
while parent_map[x] != x:
parent_map[x] = parent_map[parent_map[x]]
x = parent_map[x]
return x
def union(a: int, b: int) -> None:
ra, rb = find(a), find(b)
if ra != rb:
parent_map[rb] = ra
max_dist_sq = max_dist * max_dist
centroids = [self._box_centroid(boxes[idx]) for idx in indices]
for i_pos in range(len(indices)):
cx_i, cy_i = centroids[i_pos]
for j_pos in range(i_pos + 1, len(indices)):
cx_j, cy_j = centroids[j_pos]
dx = cx_i - cx_j
dy = cy_i - cy_j
if dx * dx + dy * dy <= max_dist_sq:
union(indices[i_pos], indices[j_pos])
groups: dict[int, list[int]] = {}
for idx in indices:
groups.setdefault(find(idx), []).append(idx)
return list(groups.values())
def _split_fire_parent_with_inner_clusters(
self,
boxes: np.ndarray,
scores: np.ndarray,
cls_ids: np.ndarray,
view_ids: np.ndarray | None = None,
inner_involve_thresh: float | None = None,
cluster_iou_thresh: float | None = None,
min_clusters: int | None = None,
min_inner_conf: float | None = None,
max_inner_area_ratio: float | None = None,
peak_dist_ratio: float | None = None,
peak_dist_min_px: float | None = None,
peak_dist_max_px: float | None = None,
peak_member_min_conf: float | None = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray | None, np.ndarray]:
if inner_involve_thresh is None:
inner_involve_thresh = self.fire_split_inner_involve_thresh
if cluster_iou_thresh is None:
cluster_iou_thresh = self.fire_split_cluster_iou_thresh
if min_clusters is None:
min_clusters = self.fire_split_min_clusters
if min_inner_conf is None:
min_inner_conf = self.fire_split_min_inner_conf
if max_inner_area_ratio is None:
max_inner_area_ratio = self.fire_split_max_inner_area_ratio
if peak_dist_ratio is None:
peak_dist_ratio = self.fire_split_peak_dist_ratio
if peak_dist_min_px is None:
peak_dist_min_px = self.fire_split_peak_dist_min_px
if peak_dist_max_px is None:
peak_dist_max_px = self.fire_split_peak_dist_max_px
if peak_member_min_conf is None:
peak_member_min_conf = self.fire_split_peak_member_min_conf
n = len(boxes)
if n <= 1:
protect = np.zeros(n, dtype=bool)
return boxes, scores, cls_ids, view_ids, protect
remove_indices: set[int] = set()
split_boxes: list[np.ndarray] = []
split_scores: list[float] = []
split_cls: list[int] = []
split_views: list[int] = []
for target_cls in self._parent_split_cls_ids():
class_indices = [i for i in range(n) if int(cls_ids[i]) == target_cls]
if len(class_indices) <= 1:
continue
class_indices_sorted = sorted(
class_indices,
key=lambda idx: self._box_area(boxes[idx]),
reverse=True,
)
for parent_idx in class_indices_sorted:
if parent_idx in remove_indices:
continue
parent_area = self._box_area(boxes[parent_idx])
parent_scale = math.sqrt(parent_area)
peak_dist = max(
peak_dist_min_px,
min(parent_scale * peak_dist_ratio, peak_dist_max_px),
)
peak_indices = [
j
for j in class_indices
if j != parent_idx
and j not in remove_indices
and self._centroid_inside_box(boxes, parent_idx, j)
]
max_inner_area = parent_area * max_inner_area_ratio
small_inner_indices = [
j
for j in peak_indices
if self._box_area(boxes[j]) <= max_inner_area
and self._mostly_inside_parent(
boxes, parent_idx, j, inner_involve_thresh
)
]
strong_clusters: list[list[int]] = []
member_indices: list[int] = []
use_small_split_outputs = False
peak_clusters: list[list[int]] = []
peak_strong: list[list[int]] = []
peak_members = [parent_idx] + [
j
for j in peak_indices
if float(scores[j]) >= peak_member_min_conf
]
if len(peak_members) >= min_clusters:
peak_clusters = self._cluster_indices_by_centroid_distance(
boxes, peak_members, peak_dist
)
peak_strong = [
cluster
for cluster in peak_clusters
if max(float(scores[m]) for m in cluster) >= min_inner_conf
]
if len(peak_strong) >= min_clusters:
strong_clusters = peak_strong
member_indices = peak_indices
if len(strong_clusters) < min_clusters and small_inner_indices:
iou_clusters = self._cluster_indices_by_box_iou(
boxes, small_inner_indices, cluster_iou_thresh
)
iou_strong = [
cluster
for cluster in iou_clusters
if max(float(scores[m]) for m in cluster) >= min_inner_conf
]
if len(iou_strong) >= min_clusters:
strong_clusters = iou_strong
member_indices = small_inner_indices
parent_blob_conf = 0.0
for cluster in peak_clusters:
if parent_idx not in cluster:
continue
non_parent = [m for m in cluster if m != parent_idx]
if non_parent:
parent_blob_conf = max(
parent_blob_conf,
max(float(scores[m]) for m in non_parent),
)
if (
len(strong_clusters) < min_clusters
and parent_blob_conf >= self.fire_split_parent_blob_conf_thresh
and len(small_inner_indices) >= min_clusters
):
small_peak_clusters = self._cluster_indices_by_centroid_distance(
boxes, small_inner_indices, peak_dist
)
small_strong = [
cluster
for cluster in small_peak_clusters
if min(
1.0,
max(float(scores[m]) for m in cluster)
+ self.fire_split_conf_boost,
)
>= min_inner_conf
]
if len(small_strong) >= min_clusters:
cluster_best = [
cluster[int(np.argmax(scores[cluster]))]
for cluster in small_strong
]
smalls_are_separate = True
for i_pos in range(len(cluster_best)):
for j_pos in range(i_pos + 1, len(cluster_best)):
if self._boxes_involve_each_other(
boxes,
cluster_best[i_pos],
cluster_best[j_pos],
self.fire_parent_involve_thresh,
):
smalls_are_separate = False
break
if not smalls_are_separate:
break
if smalls_are_separate:
strong_clusters = small_strong
member_indices = peak_indices
use_small_split_outputs = True
if len(strong_clusters) < min_clusters:
continue
remove_indices.add(parent_idx)
remove_indices.update(member_indices)
for j in class_indices:
if j in remove_indices:
continue
if self._boxes_involve_each_other(
boxes, parent_idx, j, self.fire_parent_involve_thresh
):
remove_indices.add(j)
for cluster in strong_clusters:
if use_small_split_outputs:
best_member = cluster[int(np.argmax(scores[cluster]))]
else:
cluster_candidates = [
m for m in cluster if m != parent_idx
] or cluster
best_member = cluster_candidates[
int(np.argmax(scores[cluster_candidates]))
]
boosted_conf = float(scores[best_member])
member_area = self._box_area(boxes[best_member])
if (
use_small_split_outputs
or member_area <= parent_area * max_inner_area_ratio
):
boosted_conf = min(
1.0, boosted_conf + self.fire_split_conf_boost
)
split_boxes.append(boxes[best_member].copy())
split_scores.append(boosted_conf)
split_cls.append(target_cls)
if view_ids is not None:
split_views.append(int(view_ids[best_member]))
if not remove_indices:
protect = np.zeros(n, dtype=bool)
return boxes, scores, cls_ids, view_ids, protect
keep = np.array([i not in remove_indices for i in range(n)], dtype=bool)
boxes = boxes[keep]
scores = scores[keep]
cls_ids = cls_ids[keep]
if view_ids is not None:
view_ids = view_ids[keep]
protect = np.zeros(len(boxes), dtype=bool)
if split_boxes:
split_count = len(split_boxes)
boxes = np.concatenate([boxes, np.stack(split_boxes, axis=0)], axis=0)
scores = np.concatenate(
[scores, np.asarray(split_scores, dtype=np.float32)], axis=0
)
cls_ids = np.concatenate(
[cls_ids, np.asarray(split_cls, dtype=np.int32)], axis=0
)
if view_ids is not None:
view_ids = np.concatenate(
[view_ids, np.asarray(split_views, dtype=np.int32)], axis=0
)
else:
view_ids = np.full(split_count, -1, dtype=np.int32)
protect = np.concatenate(
[protect, np.ones(split_count, dtype=bool)], axis=0
)
return boxes, scores, cls_ids, view_ids, protect
def _remove_small_group_low_conf(
self,
boxes: np.ndarray,
scores: np.ndarray,
cls_ids: np.ndarray,
min_count: int | None = None,
conf_thresh: float | None = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
if min_count is None:
min_count = self.small_group_min_count
if conf_thresh is None:
conf_thresh = self.small_group_low_conf_thresh
n = len(boxes)
if n == 0 or len(self.group_object_counts) != n:
return boxes, scores, cls_ids
keep = np.ones(n, dtype=bool)
for i in range(n):
if self.group_object_counts[i] < min_count and scores[i] < conf_thresh:
keep[i] = False
self.group_object_counts = [
count for count, k in zip(self.group_object_counts, keep) if k
]
self.group_max_confidences = [
value for value, k in zip(self.group_max_confidences, keep) if k
]
self.group_avg_confidences = [
value for value, k in zip(self.group_avg_confidences, keep) if k
]
return boxes[keep], scores[keep], cls_ids[keep]
def _apply_post_filters(
self,
boxes: np.ndarray,
scores: np.ndarray,
cls_ids: np.ndarray,
orig_size: tuple[int, int],
view_ids: np.ndarray | None = None,
) -> list[BoundingBox]:
if len(boxes) == 0:
return []
if view_ids is not None:
boxes, scores, cls_ids, view_ids = self._remove_edge_low_conf(
boxes, scores, cls_ids, orig_size, view_ids
)
else:
boxes, scores, cls_ids = self._remove_edge_low_conf(
boxes, scores, cls_ids, orig_size
)
if len(boxes) == 0:
return []
split_protect: np.ndarray | None = None
if view_ids is not None:
boxes, scores, cls_ids, view_ids = (
self._remove_isolated_low_conf_same_class(
boxes, scores, cls_ids, view_ids
)
)
boxes, scores, cls_ids, view_ids = (
self._remove_overlapping_low_conf_same_class(
boxes, scores, cls_ids, view_ids
)
)
boxes, scores, cls_ids, view_ids, split_protect = (
self._split_fire_parent_with_inner_clusters(
boxes, scores, cls_ids, view_ids
)
)
if len(boxes) == 0:
return []
boxes, scores, cls_ids, view_ids = (
self._remove_unconfirmed_single_view_low_conf(
boxes, scores, cls_ids, view_ids, protect_mask=split_protect
)
)
if len(boxes) == 0:
return []
boxes, scores, cls_ids, view_ids = self._remove_spanning_parent_fire(
boxes, scores, cls_ids, view_ids
)
if len(boxes) == 0:
return []
boxes, scores, cls_ids = self._group_overlapping_detections(
boxes, scores, cls_ids, view_ids=view_ids, record_stats=False
)
else:
boxes, scores, cls_ids = self._remove_isolated_low_conf_same_class(
boxes, scores, cls_ids
)
boxes, scores, cls_ids = self._remove_overlapping_low_conf_same_class(
boxes, scores, cls_ids
)
boxes, scores, cls_ids, _, split_protect = (
self._split_fire_parent_with_inner_clusters(
boxes, scores, cls_ids
)
)
if len(boxes) == 0:
return []
boxes, scores, cls_ids = self._remove_spanning_parent_fire(
boxes, scores, cls_ids
)
if len(boxes) == 0:
return []
boxes, scores, cls_ids = self._group_overlapping_detections(
boxes, scores, cls_ids, record_stats=True
)
boxes, scores, cls_ids = self._remove_small_group_low_conf(
boxes, scores, cls_ids
)
boxes, scores, cls_ids = self._resolve_fire_and_ext_same_class_overlap(
boxes, scores, cls_ids
)
cls_smoke = self.class_names.index("smoke")
if int(np.sum(cls_ids == cls_smoke)) >= self.smoke_cluster_merge_early_min_count:
boxes, scores, cls_ids = self._merge_overlapping_smoke_clusters(
boxes, scores, cls_ids
)
boxes, scores, cls_ids = self._resolve_smoke_overlap(
boxes, scores, cls_ids
)
boxes, scores, cls_ids = self._merge_overlapping_smoke_clusters(
boxes, scores, cls_ids
)
return self._build_results(boxes, scores, cls_ids)
@staticmethod
def _bbox_to_array(box: BoundingBox) -> np.ndarray:
return np.array([box.x1, box.y1, box.x2, box.y2], dtype=np.float32)
def _smoke_base_band_box(self, smoke: BoundingBox) -> np.ndarray:
smoke_h = smoke.y2 - smoke.y1
base_y1 = smoke.y1 + smoke_h * (1.0 - self.smoke_anchor_base_height_frac)
return np.array(
[float(smoke.x1), base_y1, float(smoke.x2), float(smoke.y2)],
dtype=np.float32,
)
def _fire_overlaps_smoke_base(
self, fire_box: np.ndarray, smoke: BoundingBox
) -> bool:
return self._boxes_involve_each_other_arrays(
fire_box,
self._smoke_base_band_box(smoke),
self.smoke_anchor_corroborate_involve_thresh,
)
def _fire_sits_under_smoke(
self, fire_box: np.ndarray, smoke: BoundingBox
) -> bool:
cx = (fire_box[0] + fire_box[2]) / 2.0
cy = (fire_box[1] + fire_box[3]) / 2.0
smoke_w = float(smoke.x2 - smoke.x1)
smoke_cx = (smoke.x1 + smoke.x2) / 2.0
if abs(cx - smoke_cx) > smoke_w * 0.55:
return False
if cy < smoke.y1 + (smoke.y2 - smoke.y1) * 0.45:
return False
return True
def _fire_centroid_in_smoke_base_band(
self, fire_box: np.ndarray, smoke: BoundingBox
) -> bool:
base = self._smoke_base_band_box(smoke)
cx = (fire_box[0] + fire_box[2]) / 2.0
cy = (fire_box[1] + fire_box[3]) / 2.0
return (
base[0] <= cx <= base[2]
and base[1] <= cy <= base[3]
)
def _corroborating_merged_tta_fire_boxes(
self,
fire_box: np.ndarray,
merged_boxes: np.ndarray,
merged_cls: np.ndarray,
smoke: BoundingBox,
) -> list[np.ndarray]:
cls_fire = self.class_names.index("fire")
involve_thresh = self.smoke_anchor_corroborate_involve_thresh
probe_area = self._box_area(fire_box)
corroborators: list[np.ndarray] = []
for i in range(len(merged_boxes)):
if int(merged_cls[i]) != cls_fire:
continue
merged_box = merged_boxes[i]
if not self._boxes_involve_each_other_arrays(
fire_box, merged_box, involve_thresh
):
continue
if not self._fire_centroid_in_smoke_base_band(merged_box, smoke):
continue
merged_area = self._box_area(merged_box)
if merged_area <= 0.0 or probe_area <= 0.0:
continue
area_ratio = probe_area / merged_area
if area_ratio > self.smoke_anchor_corroborate_area_ratio_max:
continue
if area_ratio < 1.0 / self.smoke_anchor_corroborate_area_ratio_max:
continue
corroborators.append(merged_box.copy())
return corroborators
def _snap_smoke_anchor_probe_to_corroborator(
self,
fire_box: np.ndarray,
merged_boxes: np.ndarray,
merged_cls: np.ndarray,
smoke: BoundingBox,
crop_size: tuple[int, int],
) -> np.ndarray | None:
corroborators = self._corroborating_merged_tta_fire_boxes(
fire_box, merged_boxes, merged_cls, smoke
)
if not corroborators:
return None
corroborators.sort(key=self._box_area)
for corr in corroborators:
if self._passes_smoke_anchor_probe_fire_max_size(
corr, smoke, crop_size
):
return corr.copy()
return None
def _corroborated_by_merged_tta_fire(
self,
fire_box: np.ndarray,
merged_boxes: np.ndarray,
merged_cls: np.ndarray,
smoke: BoundingBox,
) -> bool:
return bool(
self._corroborating_merged_tta_fire_boxes(
fire_box, merged_boxes, merged_cls, smoke
)
)
def _refine_smoke_anchor_probe_fire_box(
self,
probe_box: np.ndarray,
image: np.ndarray,
smoke: BoundingBox,
merged_boxes: np.ndarray,
merged_cls: np.ndarray,
crop_size: tuple[int, int],
) -> np.ndarray:
corroborators = self._corroborating_merged_tta_fire_boxes(
probe_box, merged_boxes, merged_cls, smoke
)
if not corroborators:
return probe_box
def _passes_refined(box: np.ndarray) -> bool:
return (
self._passes_smoke_anchor_probe_fire_max_size(box, smoke, crop_size)
and self._passes_smoke_anchor_probe_fire_bottom_dist(box, smoke)
and self._passes_smoke_anchor_probe_fire_color(image, box)
)
probe_area = self._box_area(probe_box)
if probe_area <= 0.0:
return probe_box
expanders: list[np.ndarray] = []
for corr in corroborators:
if not self._passes_smoke_anchor_probe_fire_max_size(
corr, smoke, crop_size
):
continue
for candidate in (
corr,
np.array(
[
min(probe_box[0], corr[0]),
min(probe_box[1], corr[1]),
max(probe_box[2], corr[2]),
max(probe_box[3], corr[3]),
],
dtype=np.float32,
),
):
if not _passes_refined(candidate):
continue
if (
candidate[0] > probe_box[0] + 1.0
and candidate[1] > probe_box[1] + 1.0
):
continue
inter = self._box_intersection_area(probe_box, candidate)
if inter / probe_area < 0.95:
continue
expanders.append(candidate)
if expanders:
refined = min(
expanders,
key=lambda box: (box[0], box[2] - box[0], box[1]),
)
else:
refined = probe_box
pad = self.smoke_anchor_corroborated_refine_pad_px
if pad > 0.0 and corroborators:
padded = refined.copy()
padded[0] = max(0.0, padded[0] - pad)
padded[1] = max(0.0, padded[1] - pad)
if _passes_refined(padded):
refined = padded
return refined
def _smoke_anchor_min_fire_size(
self,
corroborated: bool,
smoke_conf: float,
under_base: bool = False,
) -> tuple[float, float]:
if smoke_conf >= self.smoke_anchor_strong_smoke_conf and (
corroborated or under_base
):
return (
self.smoke_anchor_corroborated_min_fire_width_px,
self.smoke_anchor_corroborated_min_fire_height_px,
)
return (
self.smoke_anchor_min_fire_width_px,
self.smoke_anchor_min_fire_height_px,
)
def _passes_smoke_anchor_fire_size(
self,
fire_box: np.ndarray,
corroborated: bool,
smoke_conf: float,
under_base: bool = False,
) -> bool:
box_w = float(fire_box[2] - fire_box[0])
box_h = float(fire_box[3] - fire_box[1])
min_w, min_h = self._smoke_anchor_min_fire_size(
corroborated, smoke_conf, under_base
)
return box_w >= min_w and box_h >= min_h
def _passes_smoke_anchor_probe_fire_max_size(
self,
fire_box: np.ndarray,
smoke: BoundingBox,
crop_size: tuple[int, int],
) -> bool:
box_w = float(fire_box[2] - fire_box[0])
box_h = float(fire_box[3] - fire_box[1])
if box_w > self.smoke_anchor_max_probe_fire_width_px:
return False
if box_h > self.smoke_anchor_max_probe_fire_height_px:
return False
smoke_area = max(
1.0, float((smoke.x2 - smoke.x1) * (smoke.y2 - smoke.y1))
)
if self._box_area(fire_box) / smoke_area > self.smoke_anchor_max_probe_fire_area_ratio:
return False
crop_w, crop_h = crop_size
if crop_w > 0 and box_w / float(crop_w) > self.smoke_anchor_max_probe_crop_width_ratio:
return False
if crop_h > 0 and box_h / float(crop_h) > self.smoke_anchor_max_probe_crop_height_ratio:
return False
return True
def _passes_smoke_anchor_probe_fire_bottom_dist(
self, fire_box: np.ndarray, smoke: BoundingBox
) -> bool:
smoke_h = max(1.0, float(smoke.y2 - smoke.y1))
bottom_gap = float(smoke.y2) - float(fire_box[3])
return bottom_gap / smoke_h <= self.smoke_anchor_max_probe_fire_bottom_dist_frac
def _passes_smoke_anchor_probe_fire_color(
self, image: np.ndarray, fire_box: np.ndarray
) -> bool:
h, w = image.shape[:2]
x1 = max(0, int(math.floor(fire_box[0])))
y1 = max(0, int(math.floor(fire_box[1])))
x2 = min(w, int(math.ceil(fire_box[2])))
y2 = min(h, int(math.ceil(fire_box[3])))
if x2 <= x1 or y2 <= y1:
return False
roi = image[y1:y2, x1:x2]
if roi.size == 0:
return False
blue = roi[:, :, 0].astype(np.float32)
green = roi[:, :, 1].astype(np.float32)
red = roi[:, :, 2].astype(np.float32)
mean_r = float(np.mean(red))
max_rgb = float(
max(np.max(red), np.max(green), np.max(blue))
)
bright_frac = float(
np.mean(np.max(roi, axis=2) >= self.smoke_anchor_probe_bright_thresh)
)
if (
max_rgb >= self.smoke_anchor_probe_sparse_hot_min_max_rgb
and bright_frac >= self.smoke_anchor_probe_sparse_hot_min_bright_frac
):
return True
if mean_r < self.smoke_anchor_probe_min_mean_r:
return False
if max_rgb < self.smoke_anchor_probe_min_max_rgb:
return False
warm_mask = (red > green + 10.0) & (red > blue + 10.0)
warm_frac = float(np.mean(warm_mask))
r_minus_g = mean_r - float(np.mean(green))
if warm_frac >= self.smoke_anchor_probe_min_warm_frac:
if (
max_rgb >= self.smoke_anchor_probe_warm_min_max_rgb
or mean_r >= self.smoke_anchor_probe_warm_min_mean_r
or warm_frac >= self.smoke_anchor_probe_strong_warm_frac
):
return True
if (
bright_frac >= self.smoke_anchor_probe_min_bright_frac
and r_minus_g >= self.smoke_anchor_probe_min_r_minus_g
):
return True
if (
max_rgb >= self.smoke_anchor_probe_min_hot_max_rgb
and r_minus_g >= self.smoke_anchor_probe_min_r_minus_g
):
return True
return False
def _fire_ext_red_distribution(
self, image: np.ndarray, ext_box: np.ndarray
) -> tuple[float, float, float]:
h, w = image.shape[:2]
x1 = max(0, int(math.floor(float(ext_box[0]))))
y1 = max(0, int(math.floor(float(ext_box[1]))))
x2 = min(w, int(math.ceil(float(ext_box[2]))))
y2 = min(h, int(math.ceil(float(ext_box[3]))))
if x2 <= x1 or y2 <= y1:
return 0.0, 0.0, 0.0
roi = image[y1:y2, x1:x2]
if roi.size == 0:
return 0.0, 0.0, 0.0
blue = roi[:, :, 0].astype(np.float32)
green = roi[:, :, 1].astype(np.float32)
red = roi[:, :, 2].astype(np.float32)
red_dom = (red > green + 10.0) & (red > blue + 10.0)
return (
float(np.mean(red)),
float(np.mean(red_dom)),
float(np.mean(red - green)),
)
def _passes_fire_ext_red_color(
self, image: np.ndarray, ext_box: np.ndarray
) -> bool:
mean_r, red_dom, r_minus_g = self._fire_ext_red_distribution(
image, ext_box
)
if red_dom >= self.fire_ext_min_red_dom_frac:
return True
if (
r_minus_g >= self.fire_ext_min_r_minus_g
and mean_r >= self.fire_ext_color_min_mean_r
):
return True
return False
def _filter_fire_ext_by_red_color(
self, image: np.ndarray, results: list[BoundingBox]
) -> list[BoundingBox]:
cls_fire_ext = self.class_names.index("fire extinguisher")
max_conf = self.fire_ext_color_filter_max_conf
filtered: list[BoundingBox] = []
for box in results:
if (
box.cls_id == cls_fire_ext
and box.conf <= max_conf + 1e-4
and not self._passes_fire_ext_red_color(
image, self._bbox_to_array(box)
)
):
continue
filtered.append(box)
return filtered
def _filter_probe_fires_by_color(
self, image: np.ndarray, results: list[BoundingBox]
) -> list[BoundingBox]:
cls_fire = self.class_names.index("fire")
floor_conf = self.smoke_anchor_fire_floor_conf
filtered: list[BoundingBox] = []
for box in results:
if (
box.cls_id == cls_fire
and box.conf <= floor_conf + 1e-4
and not self._passes_smoke_anchor_probe_fire_color(
image, self._bbox_to_array(box)
)
):
continue
filtered.append(box)
return filtered
def _smoke_anchor_fire_is_duplicate(
self,
fire_box: np.ndarray,
others: list[BoundingBox],
cls_fire: int,
involve_thresh: float | None = None,
) -> bool:
if involve_thresh is None:
involve_thresh = self.smoke_anchor_corroborate_involve_thresh
for fire in others:
if fire.cls_id != cls_fire:
continue
if self._boxes_involve_each_other_arrays(
self._bbox_to_array(fire), fire_box, involve_thresh
):
return True
return False
def _collect_smoke_anchor_probe_fires(
self,
image: np.ndarray,
boxes: np.ndarray,
scores: np.ndarray,
cls_ids: np.ndarray,
smoke: BoundingBox,
merged_boxes: np.ndarray,
merged_cls: np.ndarray,
existing_fires: list[BoundingBox],
cls_fire: int,
crop_size: tuple[int, int],
) -> list[tuple[np.ndarray, float]]:
smoke_area = max(
1.0, float((smoke.x2 - smoke.x1) * (smoke.y2 - smoke.y1))
)
candidates: list[tuple[float, np.ndarray]] = []
for i in range(len(boxes)):
if int(cls_ids[i]) != cls_fire:
continue
fire_box = boxes[i]
corroborated = self._corroborated_by_merged_tta_fire(
fire_box, merged_boxes, merged_cls, smoke
)
under_smoke = self._fire_sits_under_smoke(fire_box, smoke)
in_base = self._fire_centroid_in_smoke_base_band(fire_box, smoke)
under_base = under_smoke and in_base
if not corroborated and not under_base:
continue
if not self._passes_smoke_anchor_fire_size(
fire_box, corroborated, smoke.conf, under_base
):
continue
if not self._passes_smoke_anchor_probe_fire_max_size(
fire_box, smoke, crop_size
):
if corroborated:
fire_box = self._snap_smoke_anchor_probe_to_corroborator(
fire_box, merged_boxes, merged_cls, smoke, crop_size
)
if fire_box is None:
continue
else:
continue
if not self._passes_smoke_anchor_probe_fire_bottom_dist(fire_box, smoke):
continue
if not self._passes_smoke_anchor_probe_fire_color(image, fire_box):
continue
if self._box_area(fire_box) > smoke_area * self.smoke_anchor_max_fire_area_ratio:
continue
candidates.append((float(scores[i]), fire_box.copy()))
candidates.sort(key=lambda item: item[0], reverse=True)
selected: list[tuple[np.ndarray, float]] = []
separate_thresh = self.smoke_anchor_separate_fire_involve_thresh
max_fires = self.smoke_anchor_max_fires_per_smoke
floor_conf = self.smoke_anchor_fire_floor_conf
for score, fire_box in candidates:
corroborated = self._corroborated_by_merged_tta_fire(
fire_box, merged_boxes, merged_cls, smoke
)
under_smoke = self._fire_sits_under_smoke(fire_box, smoke)
in_base = self._fire_centroid_in_smoke_base_band(fire_box, smoke)
min_conf = floor_conf
if corroborated and under_smoke and in_base:
min_conf = self.smoke_anchor_corroborated_probe_min_conf
elif not corroborated and under_smoke and in_base:
min_conf = self.smoke_anchor_probe_rescue_min_conf
if score < min_conf:
continue
if len(selected) >= max_fires:
break
if self._smoke_anchor_fire_is_duplicate(
fire_box, existing_fires, cls_fire
):
continue
overlaps_selected = False
for chosen_box, _ in selected:
if self._boxes_involve_each_other_arrays(
fire_box, chosen_box, separate_thresh
):
overlaps_selected = True
break
if overlaps_selected:
continue
selected.append((fire_box, score))
return selected
def _smoke_anchor_crop_region(
self, smoke: BoundingBox, orig_size: tuple[int, int]
) -> tuple[int, int, int, int]:
orig_w, orig_h = orig_size
smoke_w = float(smoke.x2 - smoke.x1)
smoke_h = float(smoke.y2 - smoke.y1)
pad_x = smoke_w * self.smoke_anchor_crop_width_pad_ratio
y1 = smoke.y1 + smoke_h * (1.0 - self.smoke_anchor_base_height_frac)
y2 = min(orig_h, float(smoke.y2) + self.smoke_anchor_crop_down_pad_px)
x1 = max(0, int(math.floor(smoke.x1 - pad_x)))
x2 = min(orig_w, int(math.ceil(smoke.x2 + pad_x)))
y1i = max(0, int(math.floor(y1)))
y2i = min(orig_h, max(y1i + 1, int(math.ceil(y2))))
x2 = max(x1 + 1, x2)
return x1, x2, y1i, y2i
def _smoke_anchor_probe_target_smoke(
self,
smoke: BoundingBox,
smoke_seeds: list[BoundingBox] | None,
) -> BoundingBox:
if not smoke_seeds:
return smoke
cls_smoke = self.class_names.index("smoke")
smoke_arr = self._bbox_to_array(smoke)
smoke_w = float(smoke.x2 - smoke.x1)
best_seed: BoundingBox | None = None
for seed in smoke_seeds:
if seed.cls_id != cls_smoke:
continue
if seed.conf < self.smoke_anchor_min_smoke_conf:
continue
seed_arr = self._bbox_to_array(seed)
if not self._boxes_involve_each_other_arrays(
smoke_arr, seed_arr, self.smoke_expand_corroborate_involve_thresh
):
continue
seed_w = float(seed.x2 - seed.x1)
if seed_w >= smoke_w * 0.95:
continue
if best_seed is None or seed.conf > best_seed.conf:
best_seed = seed
if best_seed is None:
return smoke
seed_w = float(best_seed.x2 - best_seed.x1)
if smoke_w > seed_w * self.smoke_anchor_expanded_width_ratio:
return best_seed
return smoke
def _probe_smoke_anchored_fire(
self,
image: np.ndarray,
results: list[BoundingBox],
orig_size: tuple[int, int],
merged_boxes: np.ndarray,
merged_cls: np.ndarray,
smoke_seeds: list[BoundingBox] | None = None,
) -> list[BoundingBox]:
if not self.use_tta:
return results
cls_fire = self.class_names.index("fire")
cls_smoke = self.class_names.index("smoke")
floor_conf = self.smoke_anchor_fire_floor_conf
added: list[BoundingBox] = []
smoke_candidates = sorted(
(
box
for box in results
if box.cls_id == cls_smoke
and box.conf >= self.smoke_anchor_min_smoke_conf
),
key=lambda box: box.conf,
reverse=True,
)[: self.smoke_anchor_max_probe_smokes]
if not smoke_candidates:
return results
for smoke in smoke_candidates:
smoke_arr = self._bbox_to_array(smoke)
fire_on_smoke_base = False
for fire in results:
if fire.cls_id != cls_fire or fire.conf < self.fire_min_conf:
continue
if self._fire_overlaps_smoke_base(
self._bbox_to_array(fire), smoke
):
fire_on_smoke_base = True
break
if fire_on_smoke_base:
continue
anchor_smoke = smoke
x1, x2, y1, y2 = self._smoke_anchor_crop_region(anchor_smoke, orig_size)
crop = image[y1:y2, x1:x2]
if crop.size == 0:
continue
crop_size = (crop.shape[1], crop.shape[0])
boxes, scores, cls_ids = self._infer_view_arrays(
crop, crop_size, flip_code=None
)
if len(boxes) == 0:
continue
boxes = boxes.copy()
boxes[:, [0, 2]] += x1
boxes[:, [1, 3]] += y1
boxes = self._clip_boxes(boxes, orig_size)
probe_fires = self._collect_smoke_anchor_probe_fires(
image,
boxes,
scores,
cls_ids,
anchor_smoke,
merged_boxes,
merged_cls,
results + added,
cls_fire,
crop_size,
)
if not probe_fires:
seed_smoke = self._smoke_anchor_probe_target_smoke(smoke, smoke_seeds)
if seed_smoke is not smoke:
x1, x2, y1, y2 = self._smoke_anchor_crop_region(
seed_smoke, orig_size
)
crop = image[y1:y2, x1:x2]
if crop.size == 0:
continue
crop_size = (crop.shape[1], crop.shape[0])
boxes, scores, cls_ids = self._infer_view_arrays(
crop, crop_size, flip_code=None
)
if len(boxes) == 0:
continue
boxes = boxes.copy()
boxes[:, [0, 2]] += x1
boxes[:, [1, 3]] += y1
boxes = self._clip_boxes(boxes, orig_size)
probe_fires = self._collect_smoke_anchor_probe_fires(
image,
boxes,
scores,
cls_ids,
seed_smoke,
merged_boxes,
merged_cls,
results + added,
cls_fire,
crop_size,
)
anchor_smoke = seed_smoke
if not probe_fires:
continue
for probe_box, probe_raw_score in probe_fires:
probe_box = self._refine_smoke_anchor_probe_fire_box(
probe_box,
image,
anchor_smoke,
merged_boxes,
merged_cls,
crop_size,
)
probe_conf = min(1.0, max(probe_raw_score, floor_conf))
added.append(
BoundingBox(
x1=int(math.floor(probe_box[0])),
y1=int(math.floor(probe_box[1])),
x2=int(math.ceil(probe_box[2])),
y2=int(math.ceil(probe_box[3])),
cls_id=cls_fire,
conf=probe_conf,
)
)
if len(added) > self.smoke_anchor_max_crop_fires:
added.sort(key=lambda box: box.conf, reverse=True)
added = added[: self.smoke_anchor_max_crop_fires]
return results + added
def _ext_probe_union_box(
self, boxes: np.ndarray, indices: list[int]
) -> np.ndarray:
cluster_boxes = boxes[indices]
return np.array(
[
float(np.min(cluster_boxes[:, 0])),
float(np.min(cluster_boxes[:, 1])),
float(np.max(cluster_boxes[:, 2])),
float(np.max(cluster_boxes[:, 3])),
],
dtype=np.float32,
)
def _ext_probe_seed_on_border(
self, seed_box: np.ndarray, orig_size: tuple[int, int]
) -> bool:
orig_w, orig_h = orig_size
tol = self.ext_probe_edge_tol
return bool(
seed_box[0] <= tol
or seed_box[1] <= tol
or seed_box[2] >= orig_w - 1 - tol
or seed_box[3] >= orig_h - 1 - tol
)
def _ext_probe_singleton_too_close_to_border(
self, seed_box: np.ndarray, orig_size: tuple[int, int]
) -> bool:
orig_w, orig_h = orig_size
margin = self.ext_probe_singleton_min_edge_margin_px
return bool(
seed_box[0] < margin
or seed_box[1] < margin
or seed_box[2] > orig_w - 1 - margin
or seed_box[3] > orig_h - 1 - margin
)
def _ext_probe_overlaps_strong_fire(
self, seed_box: np.ndarray, results: list[BoundingBox]
) -> bool:
cls_fire = self.class_names.index("fire")
for box in results:
if box.cls_id != cls_fire or box.conf < self.fire_min_conf:
continue
if self._boxes_involve_each_other_arrays(
seed_box,
self._bbox_to_array(box),
self.ext_probe_fire_exclude_involve_thresh,
):
return True
return False
def _ext_probe_near_any_fire(
self, seed_box: np.ndarray, results: list[BoundingBox]
) -> bool:
cls_fire = self.class_names.index("fire")
max_gap = self.ext_probe_near_fire_gap_px
for box in results:
if box.cls_id != cls_fire:
continue
fire_arr = self._bbox_to_array(box)
if self._box_axis_gap(seed_box, fire_arr, 0) > max_gap:
continue
if self._box_axis_gap(seed_box, fire_arr, 1) <= max_gap:
return True
return False
def _ext_probe_is_duplicate(
self,
ext_box: np.ndarray,
results: list[BoundingBox],
cls_fire_ext: int,
min_conf: float = 0.0,
) -> bool:
for box in results:
if box.cls_id != cls_fire_ext or box.conf < min_conf:
continue
if self._boxes_involve_each_other_arrays(
ext_box,
self._bbox_to_array(box),
self.ext_probe_duplicate_involve_thresh,
):
return True
return False
def _passes_ext_probe_size(
self, ext_box: np.ndarray, crop_score: float = 0.0
) -> bool:
box_w = float(ext_box[2] - ext_box[0])
box_h = float(ext_box[3] - ext_box[1])
min_w = self.ext_probe_min_width_px
if crop_score >= self.ext_probe_crop_rescue_min_conf:
min_w = min(min_w, self.ext_probe_crop_rescue_min_width_px)
return (
box_w >= min_w
and box_h >= self.ext_probe_min_height_px
and box_w <= self.ext_probe_max_width_px
and box_h <= self.ext_probe_max_height_px
)
def _passes_ext_probe_singleton_size(self, ext_box: np.ndarray) -> bool:
box_w = float(ext_box[2] - ext_box[0])
box_h = float(ext_box[3] - ext_box[1])
return (
box_w >= self.ext_probe_min_width_px
and box_h >= self.ext_probe_min_height_px
and box_w <= self.ext_probe_singleton_max_width_px
and box_h <= self.ext_probe_singleton_max_height_px
and self._box_area(ext_box) <= self.ext_probe_singleton_max_area
)
def _ext_probe_validate_seed(
self,
seed_box: np.ndarray,
best_score: float,
min_conf: float,
results: list[BoundingBox],
orig_size: tuple[int, int],
cls_fire_ext: int,
) -> bool:
if best_score < min_conf:
return False
if best_score >= self.fire_ext_min_conf:
return False
if not self._passes_ext_probe_size(seed_box):
return False
if self._ext_probe_seed_on_border(seed_box, orig_size):
return False
if self._ext_probe_overlaps_strong_fire(seed_box, results):
return False
if self._ext_probe_near_any_fire(seed_box, results):
return False
if self._ext_probe_is_duplicate(
seed_box, results, cls_fire_ext, min_conf=self.fire_ext_min_conf
):
return False
return True
def _ext_probe_crop_region(
self, seed_box: np.ndarray, orig_size: tuple[int, int]
) -> tuple[int, int, int, int]:
orig_w, orig_h = orig_size
seed_w = float(seed_box[2] - seed_box[0])
seed_h = float(seed_box[3] - seed_box[1])
pad_x = seed_w * self.ext_probe_crop_pad_ratio
pad_y = seed_h * self.ext_probe_crop_pad_ratio
x1 = max(0, int(math.floor(seed_box[0] - pad_x)))
x2 = min(orig_w, int(math.ceil(seed_box[2] + pad_x)))
y1 = max(0, int(math.floor(seed_box[1] - pad_y)))
y2 = min(orig_h, int(math.ceil(seed_box[3] + pad_y)))
min_crop = self.ext_probe_min_crop_size_px
crop_w = float(x2 - x1)
crop_h = float(y2 - y1)
if crop_w < min_crop:
cx = 0.5 * (float(seed_box[0]) + float(seed_box[2]))
half = 0.5 * min_crop
x1 = max(0, int(math.floor(cx - half)))
x2 = min(orig_w, int(math.ceil(cx + half)))
if crop_h < min_crop:
cy = 0.5 * (float(seed_box[1]) + float(seed_box[3]))
half = 0.5 * min_crop
y1 = max(0, int(math.floor(cy - half)))
y2 = min(orig_h, int(math.ceil(cy + half)))
x2 = max(x1 + 1, x2)
y2 = max(y1 + 1, y2)
return x1, x2, y1, y2
def _collect_merged_ext_probe_seeds(
self,
merged_boxes: np.ndarray,
merged_scores: np.ndarray,
merged_cls: np.ndarray,
results: list[BoundingBox],
orig_size: tuple[int, int],
) -> list[tuple[np.ndarray, float, list[int]]]:
cls_fire_ext = self.class_names.index("fire extinguisher")
ext_indices = [
i
for i in range(len(merged_boxes))
if int(merged_cls[i]) == cls_fire_ext
]
if len(ext_indices) == 0:
return []
clusters = self._cluster_indices_by_centroid_distance(
merged_boxes,
ext_indices,
self.ext_probe_cluster_centroid_dist_px,
)
seeds: list[tuple[np.ndarray, float, list[int]]] = []
for cluster in clusters:
best_idx = cluster[int(np.argmax(merged_scores[cluster]))]
best_score = float(merged_scores[best_idx])
if len(cluster) >= self.ext_probe_min_cluster_count:
if best_score < self.ext_probe_min_cluster_best_conf:
continue
seed_box = self._ext_probe_union_box(merged_boxes, cluster)
min_conf = self.ext_probe_min_cluster_best_conf
elif len(cluster) == 1:
seed_box = merged_boxes[best_idx].copy()
if not self._passes_ext_probe_singleton_size(seed_box):
continue
if self._ext_probe_singleton_too_close_to_border(seed_box, orig_size):
continue
min_conf = self.ext_probe_singleton_min_conf
else:
continue
if not self._ext_probe_validate_seed(
seed_box, best_score, min_conf, results, orig_size, cls_fire_ext
):
continue
seeds.append((seed_box, best_score, cluster))
seeds.sort(key=lambda item: item[1], reverse=True)
return seeds
def _collect_ext_probe_crop_candidates(
self,
boxes: np.ndarray,
scores: np.ndarray,
cls_ids: np.ndarray,
seed_box: np.ndarray,
cls_fire_ext: int,
) -> list[tuple[np.ndarray, float]]:
candidates: list[tuple[float, np.ndarray]] = []
for i in range(len(boxes)):
if int(cls_ids[i]) != cls_fire_ext:
continue
ext_box = boxes[i]
if not self._boxes_involve_each_other_arrays(
seed_box, ext_box, self.ext_probe_duplicate_involve_thresh
):
continue
if not self._passes_ext_probe_size(ext_box, float(scores[i])):
continue
candidates.append((float(scores[i]), ext_box.copy()))
candidates.sort(key=lambda item: item[0], reverse=True)
return [(box, score) for score, box in candidates]
def _refine_ext_probe_box(
self,
seed_box: np.ndarray,
crop_candidates: list[tuple[np.ndarray, float]],
) -> tuple[np.ndarray, float]:
if not crop_candidates:
return seed_box.copy(), 0.0
crop_box, crop_score = crop_candidates[0]
if crop_score < self.ext_probe_rescue_min_conf:
return seed_box.copy(), 0.0
refined = np.array(
[
min(float(seed_box[0]), float(crop_box[0])),
min(float(seed_box[1]), float(crop_box[1])),
max(float(seed_box[2]), float(crop_box[2])),
max(float(seed_box[3]), float(crop_box[3])),
],
dtype=np.float32,
)
if not self._passes_ext_probe_size(refined, crop_score):
return crop_box.copy(), crop_score
return refined, crop_score
def _probe_merged_anchored_fire_ext(
self,
image: np.ndarray,
results: list[BoundingBox],
orig_size: tuple[int, int],
merged_boxes: np.ndarray,
merged_scores: np.ndarray,
merged_cls: np.ndarray,
) -> list[BoundingBox]:
if not self.use_tta:
return results
cls_fire_ext = self.class_names.index("fire extinguisher")
floor_conf = self.fire_ext_min_conf
seeds = self._collect_merged_ext_probe_seeds(
merged_boxes, merged_scores, merged_cls, results, orig_size
)
if not seeds:
return results
added: list[BoundingBox] = []
for seed_box, seed_score, _cluster in seeds:
x1, x2, y1, y2 = self._ext_probe_crop_region(seed_box, orig_size)
crop = image[y1:y2, x1:x2]
crop_score = 0.0
probe_box = seed_box.copy()
if crop.size > 0:
crop_size = (crop.shape[1], crop.shape[0])
boxes, scores, cls_ids = self._infer_view_arrays(
crop, crop_size, flip_code=None
)
if len(boxes) > 0:
boxes = boxes.copy()
boxes[:, [0, 2]] += x1
boxes[:, [1, 3]] += y1
boxes = self._clip_boxes(boxes, orig_size)
crop_candidates = self._collect_ext_probe_crop_candidates(
boxes, scores, cls_ids, seed_box, cls_fire_ext
)
probe_box, crop_score = self._refine_ext_probe_box(
seed_box, crop_candidates
)
if not self._passes_ext_probe_size(probe_box, crop_score):
continue
if self._ext_probe_is_duplicate(
probe_box,
results + added,
cls_fire_ext,
min_conf=self.fire_ext_min_conf,
):
continue
probe_conf = min(
1.0, max(seed_score, crop_score, floor_conf)
)
added.append(
BoundingBox(
x1=int(math.floor(probe_box[0])),
y1=int(math.floor(probe_box[1])),
x2=int(math.ceil(probe_box[2])),
y2=int(math.ceil(probe_box[3])),
cls_id=cls_fire_ext,
conf=probe_conf,
)
)
if len(added) >= self.ext_probe_max_per_frame:
break
if not added:
return results
kept: list[BoundingBox] = []
for box in results:
if box.cls_id == cls_fire_ext and box.conf < floor_conf:
if any(
self._boxes_involve_each_other_arrays(
self._bbox_to_array(box),
self._bbox_to_array(added_box),
self.ext_probe_duplicate_involve_thresh,
)
for added_box in added
):
continue
kept.append(box)
return kept + added
def _smoke_expansion_max_deltas(
self, orig_size: tuple[int, int]
) -> tuple[float, float]:
orig_w, orig_h = orig_size
return (
float(orig_w) * self.smoke_expand_max_frame_ratio,
float(orig_h) * self.smoke_expand_max_frame_ratio,
)
def _box_axis_gap(
self, box_a: np.ndarray, box_b: np.ndarray, axis: int
) -> float:
start_a = float(box_a[axis])
end_a = float(box_a[axis + 2])
start_b = float(box_b[axis])
end_b = float(box_b[axis + 2])
if end_a >= start_b and end_b >= start_a:
return 0.0
return max(start_b - end_a, start_a - end_b)
def _smoke_probe_within_expansion_reach(
self,
seed_arr: np.ndarray,
probe_box: np.ndarray,
orig_size: tuple[int, int],
) -> bool:
max_dx, max_dy = self._smoke_expansion_max_deltas(orig_size)
if self._box_axis_gap(seed_arr, probe_box, 0) > max_dx:
return False
if self._box_axis_gap(seed_arr, probe_box, 1) > max_dy:
return False
return True
def _clamp_smoke_expansion_union(
self,
union_box: np.ndarray,
seed_arr: np.ndarray,
orig_size: tuple[int, int],
) -> np.ndarray:
max_dx, max_dy = self._smoke_expansion_max_deltas(orig_size)
union_box = union_box.copy()
union_box[0] = max(float(union_box[0]), float(seed_arr[0]) - max_dx)
union_box[2] = min(float(union_box[2]), float(seed_arr[2]) + max_dx)
union_box[1] = max(float(union_box[1]), float(seed_arr[1]) - max_dy)
union_box[3] = min(float(union_box[3]), float(seed_arr[3]) + max_dy)
return union_box
def _smoke_crop_horizontal_reach_limit(self, seed_arr: np.ndarray) -> float:
seed_w = max(1.0, float(seed_arr[2]) - float(seed_arr[0]))
return max(
32.0,
seed_w * self.smoke_expand_band_max_horizontal_gap_ratio,
)
def _clip_smoke_crop_candidate_for_union(
self,
candidate_box: np.ndarray,
seed_arr: np.ndarray,
) -> np.ndarray:
if self._probe_qualifies_for_upward_low_conf(candidate_box, seed_arr):
return candidate_box.copy()
max_h_gap = self._smoke_crop_horizontal_reach_limit(seed_arr)
clipped = candidate_box.copy()
clipped[0] = max(float(clipped[0]), float(seed_arr[0]) - max_h_gap)
clipped[2] = min(float(clipped[2]), float(seed_arr[2]) + max_h_gap)
if float(clipped[2]) <= float(clipped[0]):
clipped[0] = float(seed_arr[0])
clipped[2] = float(seed_arr[2])
return clipped
def _smoke_expansion_crop_region(
self, smoke: BoundingBox, orig_size: tuple[int, int]
) -> tuple[int, int, int, int]:
orig_w, orig_h = orig_size
max_dx, max_dy = self._smoke_expansion_max_deltas(orig_size)
x1 = max(0, int(math.floor(float(smoke.x1) - max_dx)))
x2 = min(orig_w, int(math.ceil(float(smoke.x2) + max_dx)))
y1 = max(0, int(math.floor(float(smoke.y1) - max_dy)))
y2 = min(
orig_h,
max(y1 + 1, int(math.ceil(float(smoke.y2) + max_dy))),
)
x2 = max(x1 + 1, min(orig_w, x2))
y2 = max(y1 + 1, min(orig_h, y2))
return x1, x2, y1, y2
def _smoke_expansion_uses_wide_crop(
self, smoke: BoundingBox, orig_size: tuple[int, int]
) -> bool:
orig_w, orig_h = orig_size
smoke_w = float(smoke.x2 - smoke.x1)
return (
float(smoke.y2) < orig_h * self.smoke_expand_upper_plume_max_y_ratio
and smoke_w < orig_w * self.smoke_expand_wide_seed_max_width_ratio
)
def _smoke_expansion_crop_regions(
self, smoke: BoundingBox, orig_size: tuple[int, int]
) -> list[tuple[int, int, int, int]]:
primary = self._smoke_expansion_crop_region(smoke, orig_size)
regions = [primary]
if not self._smoke_expansion_uses_wide_crop(smoke, orig_size):
return regions
orig_w, orig_h = orig_size
px1, px2, py1, py2 = primary
max_dy = self._smoke_expansion_max_deltas(orig_size)[1]
smoke_h = max(1.0, float(smoke.y2 - smoke.y1))
slice_y1 = max(
py1,
int(math.floor(float(smoke.y1) - min(smoke_h * 0.15, max_dy))),
)
slice_y2 = min(
py2,
max(slice_y1 + 1, int(math.ceil(float(smoke.y2) + max_dy))),
)
left_x2 = max(px1 + 1, min(px2, int(math.ceil(float(smoke.x2)))))
right_x1 = min(px2 - 1, max(px1, int(math.floor(float(smoke.x1)))))
min_slice_w = max(40, int(orig_w * 0.08))
if left_x2 - px1 >= min_slice_w:
regions.append((px1, left_x2, slice_y1, slice_y2))
if px2 - right_x1 >= min_slice_w:
regions.append((right_x1, px2, slice_y1, slice_y2))
return regions
def _smoke_corroborates_upward_diagonal_plume(
self,
probe_box: np.ndarray,
seed_arr: np.ndarray,
crop_x1: int,
crop_x2: int,
) -> bool:
if not self._probe_extends_smoke_upward(probe_box, seed_arr):
return False
smoke_h = max(1.0, float(seed_arr[3]) - seed_arr[1])
smoke_w = max(1.0, float(seed_arr[2]) - seed_arr[0])
if float(probe_box[3]) < float(seed_arr[1]) - smoke_h * 0.15:
return False
cy = (float(probe_box[1]) + float(probe_box[3])) / 2.0
if cy > float(seed_arr[1]) + smoke_h * 0.35:
return False
gap_x = self._box_axis_gap(probe_box, seed_arr, 0)
max_gap = max(
32.0,
smoke_w * self.smoke_expand_upward_diagonal_max_gap_ratio,
)
if gap_x > max_gap:
return False
cx = (float(probe_box[0]) + float(probe_box[2])) / 2.0
return float(crop_x1) <= cx <= float(crop_x2)
def _smoke_corroborates_expansion_seed(
self,
probe_box: np.ndarray,
seed: BoundingBox,
seed_arr: np.ndarray,
crop_x1: int,
crop_x2: int,
) -> bool:
if self._boxes_involve_each_other_arrays(
probe_box,
seed_arr,
self.smoke_expand_corroborate_involve_thresh,
):
return True
if self._smoke_corroborates_upward_diagonal_plume(
probe_box, seed_arr, crop_x1, crop_x2
):
return True
smoke_h = max(1.0, float(seed.y2 - seed.y1))
band_pad = smoke_h * self.smoke_expand_vertical_band_pad_ratio
cy = (float(probe_box[1]) + float(probe_box[3])) / 2.0
cx = (float(probe_box[0]) + float(probe_box[2])) / 2.0
if cy < float(seed.y1) - band_pad or cy > float(seed.y2) + band_pad:
return False
gap_x = self._box_axis_gap(probe_box, seed_arr, 0)
seed_w = max(1.0, float(seed.x2 - seed.x1))
max_band_gap = max(
32.0,
seed_w * self.smoke_expand_band_max_horizontal_gap_ratio,
)
if gap_x > max_band_gap:
return False
return float(crop_x1) <= cx <= float(crop_x2)
def _smoke_box_border_strip(self, bw: int, bh: int) -> int:
return max(
2,
min(
int(min(bw, bh) * self.smoke_expand_border_strip_ratio),
bw // 3,
bh // 3,
),
)
def _smoke_expansion_side_border_rgb(
self,
image: np.ndarray,
x1: int,
y1: int,
x2: int,
y2: int,
side: str,
) -> np.ndarray | None:
bw = x2 - x1
bh = y2 - y1
if bw < 2 or bh < 2:
return None
strip = self._smoke_box_border_strip(bw, bh)
if side == "left":
block = image[y1:y2, x1 : x1 + strip]
elif side == "right":
block = image[y1:y2, x2 - strip : x2]
elif side == "top":
block = image[y1 : y1 + strip, x1:x2]
else:
block = image[y2 - strip : y2, x1:x2]
if block.size == 0:
return None
return np.array(
[
float(np.mean(block[:, :, 2])),
float(np.mean(block[:, :, 1])),
float(np.mean(block[:, :, 0])),
],
dtype=np.float32,
)
def _smoke_expansion_border_reference_rgb(
self, image: np.ndarray, seed: BoundingBox
) -> np.ndarray | None:
h, w = image.shape[:2]
x1 = max(0, int(math.floor(seed.x1)))
y1 = max(0, int(math.floor(seed.y1)))
x2 = min(w, int(math.ceil(seed.x2)))
y2 = min(h, int(math.ceil(seed.y2)))
block_means: list[np.ndarray] = []
for side in ("top", "bottom", "left", "right"):
rgb = self._smoke_expansion_side_border_rgb(image, x1, y1, x2, y2, side)
if rgb is not None:
block_means.append(rgb)
if not block_means:
return None
return np.mean(block_means, axis=0)
def _smoke_expansion_tta_view_corroborated(
self,
index: int,
boxes: np.ndarray,
cls_ids: np.ndarray,
view_ids: np.ndarray,
cls_smoke: int,
) -> bool:
n = len(boxes)
if n <= 1 or int(cls_ids[index]) != cls_smoke:
return False
iou = self._compute_iou_matrix(boxes)
other = np.arange(n) != index
other_view = view_ids != view_ids[index]
same_class = cls_ids == cls_smoke
return bool(
np.any(
other
& other_view
& same_class
& (iou[index] >= self.smoke_expand_tta_view_iou_thresh)
)
)
def _probe_extends_smoke_upward(
self, probe_box: np.ndarray, seed_arr: np.ndarray, margin: float = 2.0
) -> bool:
return float(probe_box[1]) < float(seed_arr[1]) - margin
def _probe_extends_smoke_downward(
self, probe_box: np.ndarray, seed_arr: np.ndarray, margin: float = 2.0
) -> bool:
return float(probe_box[3]) > float(seed_arr[3]) + margin
def _probe_qualifies_for_upward_low_conf(
self, probe_box: np.ndarray, seed_arr: np.ndarray, margin: float = 2.0
) -> bool:
if not self._probe_extends_smoke_upward(probe_box, seed_arr, margin):
return False
return not self._probe_extends_smoke_downward(probe_box, seed_arr, margin)
def _probe_extends_smoke_horizontally(
self, probe_box: np.ndarray, seed_arr: np.ndarray, margin: float = 2.0
) -> bool:
return (
float(probe_box[0]) < float(seed_arr[0]) - margin
or float(probe_box[2]) > float(seed_arr[2]) + margin
)
def _probe_extends_smoke_seed(
self, probe_box: np.ndarray, seed_arr: np.ndarray, margin: float = 2.0
) -> bool:
return (
self._probe_extends_smoke_horizontally(probe_box, seed_arr, margin)
or self._probe_extends_smoke_upward(probe_box, seed_arr, margin)
or float(probe_box[3]) > float(seed_arr[3]) + margin
)
def _smoke_expansion_tta_candidate_conf_ok(
self, score: float, candidate_box: np.ndarray, seed_arr: np.ndarray
) -> bool:
if self._probe_qualifies_for_upward_low_conf(candidate_box, seed_arr):
if score >= self.smoke_expand_min_upward_tta_conf:
return True
if score >= self.smoke_expand_min_upward_tta_involve_conf:
return self._boxes_involve_each_other_arrays(
candidate_box,
seed_arr,
self.smoke_expand_corroborate_involve_thresh,
)
return False
if score < self.smoke_expand_min_tta_smoke_conf:
return False
if self._probe_extends_smoke_horizontally(
candidate_box, seed_arr
) or self._probe_extends_smoke_downward(candidate_box, seed_arr):
return score >= self.smoke_expand_min_tta_extend_conf
return True
def _smoke_expansion_crop_candidate_conf_ok(
self, score: float, candidate_box: np.ndarray, seed_arr: np.ndarray
) -> bool:
if (
self._probe_extends_smoke_horizontally(candidate_box, seed_arr)
and self._probe_extends_smoke_downward(candidate_box, seed_arr)
and score < self.smoke_expand_min_bidi_crop_conf
):
return False
if self._probe_qualifies_for_upward_low_conf(candidate_box, seed_arr):
return score >= self.smoke_expand_min_upward_crop_conf
if score < self.smoke_expand_min_probe_smoke_conf:
return False
if self._probe_extends_smoke_horizontally(
candidate_box, seed_arr
) or self._probe_extends_smoke_downward(candidate_box, seed_arr):
return score >= self.smoke_expand_min_extend_probe_conf
return True
def _smoke_expand_mean_rgb_distance(
self, image: np.ndarray, box: np.ndarray, ref_rgb: np.ndarray
) -> float:
h, w = image.shape[:2]
x1 = max(0, int(math.floor(float(box[0]))))
y1 = max(0, int(math.floor(float(box[1]))))
x2 = min(w, int(math.ceil(float(box[2]))))
y2 = min(h, int(math.ceil(float(box[3]))))
if x2 <= x1 or y2 <= y1:
return float("inf")
roi = image[y1:y2, x1:x2]
if roi.size == 0:
return float("inf")
mean_rgb = np.array(
[
float(np.mean(roi[:, :, 2])),
float(np.mean(roi[:, :, 1])),
float(np.mean(roi[:, :, 0])),
],
dtype=np.float32,
)
return float(np.linalg.norm(mean_rgb - ref_rgb))
def _passes_smoke_expand_probe_color(
self,
image: np.ndarray,
probe_box: np.ndarray,
ref_rgb: np.ndarray,
) -> bool:
return (
self._smoke_expand_mean_rgb_distance(image, probe_box, ref_rgb)
<= self.smoke_expand_max_color_dist
)
def _smoke_expansion_union_candidate(
self,
union_box: np.ndarray,
seed_arr: np.ndarray,
candidate_box: np.ndarray,
candidate_score: float,
max_conf: float,
) -> tuple[np.ndarray, float, bool]:
union_box = union_box.copy()
union_box[0] = min(union_box[0], candidate_box[0])
union_box[1] = min(union_box[1], candidate_box[1])
union_box[2] = max(union_box[2], candidate_box[2])
union_box[3] = max(union_box[3], candidate_box[3])
margin = 2.0
found_extra = (
float(candidate_box[0]) < float(seed_arr[0]) - margin
or float(candidate_box[2]) > float(seed_arr[2]) + margin
or float(candidate_box[1]) < float(seed_arr[1]) - margin
or float(candidate_box[3]) > float(seed_arr[3]) + margin
)
return union_box, max(max_conf, candidate_score), found_extra
def _collect_smoke_expansion_cluster(
self,
image: np.ndarray,
seed: BoundingBox,
orig_size: tuple[int, int],
crop_x1: int,
crop_x2: int,
ref_rgb: np.ndarray | None,
tta_boxes: np.ndarray,
tta_scores: np.ndarray,
tta_cls: np.ndarray,
tta_view_ids: np.ndarray | None,
crop_boxes: np.ndarray | None = None,
crop_scores: np.ndarray | None = None,
crop_cls: np.ndarray | None = None,
) -> tuple[np.ndarray, float] | None:
cls_smoke = self.class_names.index("smoke")
seed_arr = self._bbox_to_array(seed)
union_box = seed_arr.copy()
max_conf = float(seed.conf)
found_extra = False
for i in range(len(tta_boxes)):
if int(tta_cls[i]) != cls_smoke:
continue
candidate_box = tta_boxes[i]
if not self._smoke_expansion_tta_candidate_conf_ok(
float(tta_scores[i]), candidate_box, seed_arr
):
continue
if tta_view_ids is not None and not self._smoke_expansion_tta_view_corroborated(
i, tta_boxes, tta_cls, tta_view_ids, cls_smoke
):
continue
if not self._smoke_corroborates_expansion_seed(
candidate_box, seed, seed_arr, crop_x1, crop_x2
):
continue
if not self._smoke_probe_within_expansion_reach(
seed_arr, candidate_box, orig_size
):
continue
union_box, max_conf, extra = self._smoke_expansion_union_candidate(
union_box, seed_arr, candidate_box, float(tta_scores[i]), max_conf
)
found_extra = found_extra or extra
if (
crop_boxes is not None
and crop_scores is not None
and crop_cls is not None
and ref_rgb is not None
and len(crop_boxes) > 0
):
for i in range(len(crop_boxes)):
if int(crop_cls[i]) != cls_smoke:
continue
candidate_box = crop_boxes[i]
if not self._smoke_expansion_crop_candidate_conf_ok(
float(crop_scores[i]), candidate_box, seed_arr
):
continue
if not self._smoke_corroborates_expansion_seed(
candidate_box, seed, seed_arr, crop_x1, crop_x2
):
continue
if not self._smoke_probe_within_expansion_reach(
seed_arr, candidate_box, orig_size
):
continue
if not self._passes_smoke_expand_probe_color(
image, candidate_box, ref_rgb
):
continue
candidate_box = self._clip_smoke_crop_candidate_for_union(
candidate_box, seed_arr
)
union_box, max_conf, extra = self._smoke_expansion_union_candidate(
union_box,
seed_arr,
candidate_box,
float(crop_scores[i]),
max_conf,
)
found_extra = found_extra or extra
union_box = self._clamp_smoke_expansion_union(union_box, seed_arr, orig_size)
margin = 2.0
refined_seed = (
float(union_box[0]) < float(seed_arr[0]) - margin
or float(union_box[2]) > float(seed_arr[2]) + margin
or float(union_box[1]) < float(seed_arr[1]) - margin
or float(union_box[3]) > float(seed_arr[3]) + margin
or float(union_box[0]) > float(seed_arr[0]) + margin
or float(union_box[2]) < float(seed_arr[2]) - margin
or float(union_box[1]) > float(seed_arr[1]) + margin
or float(union_box[3]) < float(seed_arr[3]) - margin
)
if not found_extra and not refined_seed:
return None
return union_box, max_conf
def _smoke_expansion_seed_is_complete(
self,
seed: BoundingBox,
orig_size: tuple[int, int],
tta_boxes: np.ndarray,
tta_cls: np.ndarray,
tta_view_ids: np.ndarray | None,
) -> bool:
orig_w, orig_h = orig_size
seed_w = float(seed.x2 - seed.x1)
width_ratio = seed_w / max(1.0, float(orig_w))
y1_ratio = float(seed.y1) / max(1.0, float(orig_h))
if y1_ratio > self.smoke_expand_skip_max_y1_ratio:
return False
if width_ratio < self.smoke_expand_skip_min_width_ratio:
return False
if tta_view_ids is None or len(tta_boxes) == 0:
return False
cls_smoke = self.class_names.index("smoke")
seed_arr = self._bbox_to_array(seed)
matched_views: set[int] = set()
for i in range(len(tta_boxes)):
if int(tta_cls[i]) != cls_smoke:
continue
if self._boxes_involve_each_other_arrays(
tta_boxes[i],
seed_arr,
self.smoke_expand_corroborate_involve_thresh,
):
matched_views.add(int(tta_view_ids[i]))
if len(matched_views) < 2:
return False
return seed.conf >= self.smoke_expand_skip_min_conf
@staticmethod
def _smoke_expansion_union_extended_seed(
seed_arr: np.ndarray, union_box: np.ndarray, margin: float = 2.0
) -> bool:
return (
float(union_box[0]) < float(seed_arr[0]) - margin
or float(union_box[2]) > float(seed_arr[2]) + margin
or float(union_box[1]) < float(seed_arr[1]) - margin
or float(union_box[3]) > float(seed_arr[3]) + margin
)
def _probe_smoke_expansion(
self,
image: np.ndarray,
results: list[BoundingBox],
orig_size: tuple[int, int],
tta_boxes: np.ndarray,
tta_scores: np.ndarray,
tta_cls: np.ndarray,
tta_view_ids: np.ndarray | None,
) -> list[BoundingBox]:
if not self.use_tta:
return results
cls_smoke = self.class_names.index("smoke")
seeds = sorted(
(
box
for box in results
if box.cls_id == cls_smoke
and box.conf >= self.smoke_expand_min_seed_conf
),
key=lambda box: box.conf,
reverse=True,
)
if not seeds:
return results
seeds = seeds[: self.smoke_expand_max_seeds]
absorbed: set[int] = set()
replacements: dict[int, BoundingBox] = {}
for seed in seeds:
seed_key = id(seed)
if seed_key in absorbed:
continue
skip_crops_only = self._smoke_expansion_seed_is_complete(
seed, orig_size, tta_boxes, tta_cls, tta_view_ids
)
crop_regions = self._smoke_expansion_crop_regions(seed, orig_size)
primary_x1 = crop_regions[0][0]
primary_x2 = crop_regions[0][1]
ref_rgb = self._smoke_expansion_border_reference_rgb(image, seed)
expanded = self._collect_smoke_expansion_cluster(
image,
seed,
orig_size,
primary_x1,
primary_x2,
ref_rgb,
tta_boxes,
tta_scores,
tta_cls,
tta_view_ids,
None,
None,
None,
)
seed_arr = self._bbox_to_array(seed)
orig_h = orig_size[1]
upward_open = float(seed.y1) / max(1.0, float(orig_h)) > (
self.smoke_expand_skip_max_y1_ratio
)
if skip_crops_only:
pass
elif (
expanded is not None
and self._smoke_expansion_union_extended_seed(seed_arr, expanded[0])
and not upward_open
):
pass
else:
crop_boxes_list: list[np.ndarray] = []
crop_scores_list: list[np.ndarray] = []
crop_cls_list: list[np.ndarray] = []
for x1, x2, y1, y2 in crop_regions:
crop = image[y1:y2, x1:x2]
if crop.size == 0 or crop.shape[0] < 2 or crop.shape[1] < 2:
continue
crop_size = (crop.shape[1], crop.shape[0])
boxes, scores, cls_ids = self._infer_view_arrays(
crop, crop_size, flip_code=None
)
if len(boxes) == 0:
continue
boxes = boxes.copy()
boxes[:, [0, 2]] += x1
boxes[:, [1, 3]] += y1
boxes = self._clip_boxes(boxes, orig_size)
crop_boxes_list.append(boxes)
crop_scores_list.append(scores)
crop_cls_list.append(cls_ids)
merged_crop_boxes: np.ndarray | None = None
merged_crop_scores: np.ndarray | None = None
merged_crop_cls: np.ndarray | None = None
if crop_boxes_list and ref_rgb is not None:
merged_crop_boxes = np.concatenate(crop_boxes_list, axis=0)
merged_crop_scores = np.concatenate(crop_scores_list, axis=0)
merged_crop_cls = np.concatenate(crop_cls_list, axis=0)
crop_expanded = self._collect_smoke_expansion_cluster(
image,
seed,
orig_size,
primary_x1,
primary_x2,
ref_rgb,
tta_boxes,
tta_scores,
tta_cls,
tta_view_ids,
merged_crop_boxes,
merged_crop_scores,
merged_crop_cls,
)
if crop_expanded is not None:
expanded = crop_expanded
if expanded is None:
continue
union_box, max_conf = expanded
seed_arr = self._bbox_to_array(seed)
for other in results:
if other.cls_id != cls_smoke or other is seed:
continue
other_key = id(other)
if other_key in absorbed:
continue
if other_key in replacements:
other_arr = self._bbox_to_array(replacements[other_key])
else:
other_arr = self._bbox_to_array(other)
if self._boxes_involve_each_other_arrays(
other_arr,
seed_arr,
self.smoke_expand_corroborate_involve_thresh,
) or self._boxes_involve_each_other_arrays(
other_arr,
union_box,
self.smoke_expand_corroborate_involve_thresh,
):
union_box[0] = min(union_box[0], other_arr[0])
union_box[1] = min(union_box[1], other_arr[1])
union_box[2] = max(union_box[2], other_arr[2])
union_box[3] = max(union_box[3], other_arr[3])
other_conf = (
float(replacements[other_key].conf)
if other_key in replacements
else float(other.conf)
)
max_conf = max(max_conf, other_conf)
absorbed.add(other_key)
expanded_smoke = BoundingBox(
x1=int(math.floor(union_box[0])),
y1=int(math.floor(union_box[1])),
x2=int(math.ceil(union_box[2])),
y2=int(math.ceil(union_box[3])),
cls_id=cls_smoke,
conf=min(1.0, max_conf),
)
replacements[seed_key] = expanded_smoke
if not replacements and not absorbed:
return results
updated: list[BoundingBox] = []
for box in results:
box_key = id(box)
if box_key in absorbed:
continue
if box_key in replacements:
updated.append(replacements[box_key])
continue
updated.append(box)
return updated
def _infer_view_arrays(
self,
image: np.ndarray,
orig_size: tuple[int, int],
flip_code: int | None = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
if flip_code is not None:
image = cv2.flip(image, flip_code)
decode_size = (image.shape[1], image.shape[0])
input_tensor, ratio, pad, _ = self._preprocess(image)
expected = (1, 3, self.input_height, self.input_width)
if input_tensor.shape != expected:
raise ValueError(
f"Bad input tensor shape={input_tensor.shape}, expected={expected}"
)
outputs = self.session.run(self.output_names, {self.input_name: input_tensor})
boxes, scores, cls_ids = self._decode_preds_to_arrays(
outputs[0], ratio, pad, decode_size
)
if flip_code is not None and len(boxes) > 0:
boxes = self._map_flipped_boxes_arrays(boxes, decode_size, flip_code)
return boxes, scores, cls_ids
def _collect_merged_arrays(
self, image: np.ndarray
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, tuple[int, int]]:
orig_size = (image.shape[1], image.shape[0])
if self.use_tta:
view_flip_codes: tuple[int | None, ...] = (None, 1)
else:
view_flip_codes = (None,)
all_boxes: list[np.ndarray] = []
all_scores: list[np.ndarray] = []
all_cls: list[np.ndarray] = []
all_view_ids: list[np.ndarray] = []
view_id = 0
for flip_code in view_flip_codes:
boxes, scores, cls_ids = self._infer_view_arrays(
image, orig_size, flip_code
)
if len(boxes) > 0:
all_boxes.append(boxes)
all_scores.append(scores)
all_cls.append(cls_ids)
all_view_ids.append(
np.full(len(boxes), view_id, dtype=np.int32)
)
view_id += 1
if not all_boxes:
return (
np.empty((0, 4), dtype=np.float32),
np.empty((0,), dtype=np.float32),
np.empty((0,), dtype=np.int32),
np.empty((0,), dtype=np.int32),
orig_size,
)
return (
np.concatenate(all_boxes, axis=0),
np.concatenate(all_scores, axis=0),
np.concatenate(all_cls, axis=0),
np.concatenate(all_view_ids, axis=0),
orig_size,
)
def _decode_final_dets(
self,
preds: np.ndarray,
ratio: float,
pad: tuple[float, float],
orig_size: tuple[int, int],
) -> list[BoundingBox]:
boxes, scores, cls_ids = self._decode_preds_to_arrays(
preds, ratio, pad, orig_size
)
return self._apply_post_filters(boxes, scores, cls_ids, orig_size)
def _postprocess(
self,
output: np.ndarray,
ratio: float,
pad: tuple[float, float],
orig_size: tuple[int, int],
) -> list[BoundingBox]:
return self._decode_final_dets(output, ratio, pad, orig_size)
@staticmethod
def _validate_image(image: np.ndarray) -> np.ndarray:
if image is None:
raise ValueError("Input image is None")
if not isinstance(image, np.ndarray):
raise TypeError(f"Input is not numpy array: {type(image)}")
if image.ndim != 3:
raise ValueError(f"Expected HWC image, got shape={image.shape}")
if image.shape[0] <= 0 or image.shape[1] <= 0:
raise ValueError(f"Invalid image shape={image.shape}")
if image.shape[2] != 3:
raise ValueError(f"Expected 3 channels, got shape={image.shape}")
if image.dtype != np.uint8:
image = image.astype(np.uint8)
return image
def _predict_single(self, image: np.ndarray) -> list[BoundingBox]:
image = self._validate_image(image)
boxes, scores, cls_ids, view_ids, orig_size = self._collect_merged_arrays(
image
)
results = self._apply_post_filters(
boxes,
scores,
cls_ids,
orig_size,
view_ids,
)
cls_smoke = self.class_names.index("smoke")
has_expandable_smoke = any(
box.cls_id == cls_smoke and box.conf >= self.smoke_expand_min_seed_conf
for box in results
)
smoke_seeds = [
BoundingBox(
x1=box.x1,
y1=box.y1,
x2=box.x2,
y2=box.y2,
cls_id=box.cls_id,
conf=box.conf,
)
for box in results
if box.cls_id == cls_smoke
]
if has_expandable_smoke:
results = self._probe_smoke_expansion(
image, results, orig_size, boxes, scores, cls_ids, view_ids
)
has_anchor_smoke = any(
box.cls_id == cls_smoke
and box.conf >= self.smoke_anchor_min_smoke_conf
for box in results
)
if has_anchor_smoke:
results = self._probe_smoke_anchored_fire(
image, results, orig_size, boxes, cls_ids, smoke_seeds=smoke_seeds
)
results = self._probe_merged_anchored_fire_ext(
image, results, orig_size, boxes, scores, cls_ids
)
results = self._filter_probe_fires_by_color(image, results)
results = self._filter_fire_ext_by_red_color(image, results)
return self._filter_results_min_conf(results)
def predict_batch(
self,
batch_images: list[ndarray],
offset: int,
n_keypoints: int,
) -> list[TVFrameResult]:
results: list[TVFrameResult] = []
for frame_number_in_batch, image in enumerate(batch_images):
try:
boxes = self._predict_single(image)
except Exception as e:
print(
f"⚠️ Inference failed for frame "
f"{offset + frame_number_in_batch}: {e}"
)
boxes = []
results.append(
TVFrameResult(
frame_id=offset + frame_number_in_batch,
boxes=boxes,
keypoints=[(0, 0) for _ in range(max(0, int(n_keypoints)))],
)
)
return results