scorevision: push artifact

miner.py

@@ -1,17 +1,22 @@
+"""Plate-detection miner — v3 "plate_v3 + tight softnms".
+
+Base weights: plate_v3 (YOLO26s fine-tuned on Roboflow-filtered + 10x live pseudo-GT,
+resumed from plate_v2). fp16 end2end ONNX, static 1x3x1280x1280, ~19.4 MB.
+
+Inference pipeline (tuned per bench_v2.py on 184-shard pool):
+  - Single full-image pass with soft-NMS + hflip TTA
+  - Tight preset: conf=0.30, iou=0.45, sigma=0.5, max_det=16
+  - No tile fallback (v3's mAP=0.973 is already high enough; tiles only add FPs)
+
+Bench on 184-shard live pseudo-GT pool (/mnt/shadeform-data/plate_research/live_gt/):
+  gated=0.441  mAP=0.973  fp/img=0.29  ms_med=152  ms_p95=161
+Compared to:
+  plate_v2 best:     gated=0.424
+  hermestech best:   gated=0.422
+  5GRAm best:        gated=0.401
+"""
 from pathlib import Path
 import math
-import os
-import glob
-import site
-import ctypes
-
-# Preload pip-installed NVIDIA cuDNN so onnxruntime can use CUDAExecutionProvider
-for sp in site.getsitepackages():
-    for d in glob.glob(os.path.join(sp, 'nvidia', '*', 'lib')):
-        os.environ['LD_LIBRARY_PATH'] = d + ':' + os.environ.get('LD_LIBRARY_PATH', '')
-        _cudnn = os.path.join(d, 'libcudnn.so.9')
-        if os.path.exists(_cudnn):
-            ctypes.CDLL(_cudnn, mode=ctypes.RTLD_GLOBAL)
 
 import cv2
 import numpy as np
@@ -35,136 +40,305 @@ class TVFrameResult(BaseModel):
     keypoints: list[tuple[int, int]]
 
 
+SIZE = 1280
+
+
 class Miner:
     def __init__(self, path_hf_repo: Path) -> None:
-        self.path_hf_repo = path_hf_repo
-        self.class_names = ['numberplate']
-        self.session = ort.InferenceSession(
-            str(path_hf_repo / "weights.onnx"),
-            providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
-        )
+        model_path = path_hf_repo / "weights.onnx"
+        cn_path = model_path.with_name("class_names.txt")
+        if cn_path.is_file():
+            lines = cn_path.read_text(encoding="utf-8").splitlines()
+            self.class_names = [
+                ln.strip()
+                for ln in lines
+                if ln.strip() and not ln.strip().startswith("#")
+            ]
+        else:
+            self.class_names = ["numberplate"]
+        print("ORT version:", ort.__version__)
+
+        try:
+            ort.preload_dlls()
+            print("onnxruntime.preload_dlls() success")
+        except Exception as e:
+            print(f"preload_dlls failed: {e}")
+
+        print("ORT available providers BEFORE session:", ort.get_available_providers())
+
+        try:
+            import torch
+            if torch.cuda.is_available():
+                print(f"GPU: {torch.cuda.get_device_name(0)}")
+                print(f"GPU memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB")
+            else:
+                print("GPU: CUDA not available via torch")
+        except Exception as e:
+            print(f"GPU detection failed: {e}")
+
+        sess_options = ort.SessionOptions()
+        sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
+
+        try:
+            self.session = ort.InferenceSession(
+                str(model_path),
+                sess_options=sess_options,
+                providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
+            )
+            print("Created ORT session with preferred CUDA provider list")
+        except Exception as e:
+            print(f"CUDA session creation failed, falling back to CPU: {e}")
+            self.session = ort.InferenceSession(
+                str(model_path),
+                sess_options=sess_options,
+                providers=["CPUExecutionProvider"],
+            )
+
+        print("ORT session providers:", self.session.get_providers())
+
+        for inp in self.session.get_inputs():
+            print("INPUT:", inp.name, inp.shape, inp.type)
+        for out in self.session.get_outputs():
+            print("OUTPUT:", out.name, out.shape, out.type)
+
         self.input_name = self.session.get_inputs()[0].name
-        input_shape = self.session.get_inputs()[0].shape
-        self.input_size = int(input_shape[2])  # 1280
-        self.conf_threshold = 0.20
-        self.iou_threshold = 0.5
-        self.tile_overlap = 0.2
+        self.output_names = [o.name for o in self.session.get_outputs()]
+        self.input_shape = self.session.get_inputs()[0].shape
+        # plate_v3 export is fp16 static [1,3,1280,1280]
+        self.input_dtype = (
+            np.float16
+            if "float16" in self.session.get_inputs()[0].type
+            else np.float32
+        )
+
+        self.input_height = self._safe_dim(self.input_shape[2], default=SIZE)
+        self.input_width = self._safe_dim(self.input_shape[3], default=SIZE)
+
+        # Tuned preset for plate_v3 (from bench_v2.py, 184-shard live pool).
+        # Best gated=0.441 AND lowest fp/img=0.29 AND tight ms_p95=161.
+        self.conf_thres = 0.30
+        self.iou_thres = 0.45
+        self.sigma = 0.5
+        self.max_det = 16
+        self.use_tta = True
+
+        print(f"ONNX model loaded from: {model_path}")
+        print(f"ONNX providers: {self.session.get_providers()}")
+        print(f"ONNX input: name={self.input_name}, shape={self.input_shape}, dtype={self.input_dtype}")
+        print(f"Preset: conf={self.conf_thres} iou={self.iou_thres} sigma={self.sigma} max_det={self.max_det}")
 
     def __repr__(self) -> str:
-        return f"ONNX Miner session={type(self.session).__name__} classes={len(self.class_names)}"
-
-    def _letterbox(self, img: ndarray) -> tuple[ndarray, float, int, int]:
-        h, w = img.shape[:2]
-        r = min(self.input_size / h, self.input_size / w)
-        nw, nh = int(w * r), int(h * r)
-        resized = cv2.resize(img, (nw, nh), interpolation=cv2.INTER_LINEAR)
-        canvas = np.full((self.input_size, self.input_size, 3), 114, dtype=np.uint8)
-        dx = (self.input_size - nw) // 2
-        dy = (self.input_size - nh) // 2
-        canvas[dy:dy+nh, dx:dx+nw] = resized
-        return canvas, r, dx, dy
-
-    def _run_single(self, img: ndarray) -> list[tuple[float, float, float, float, float]]:
-        h, w = img.shape[:2]
-        canvas, r, dx, dy = self._letterbox(img)
-        blob = (canvas.astype(np.float32) / 255.0).transpose(2, 0, 1)[np.newaxis]
-        out = self.session.run(None, {self.input_name: blob})[0][0]
-        dets = []
-        for row in out:
-            x1, y1, x2, y2, conf, cls = row
-            if conf < self.conf_threshold:
-                continue
-            dets.append((
-                float(conf),
-                (x1 - dx) / r,
-                (y1 - dy) / r,
-                (x2 - dx) / r,
-                (y2 - dy) / r,
-            ))
-        return dets
-
-    def _nms(self, dets: list[tuple[float, float, float, float, float]]) -> list[tuple[float, float, float, float, float]]:
-        if not dets:
+        return (
+            f"ONNXRuntime(session={type(self.session).__name__}, "
+            f"providers={self.session.get_providers()})"
+        )
+
+    @staticmethod
+    def _safe_dim(value, default: int) -> int:
+        return value if isinstance(value, int) and value > 0 else default
+
+    # ---------- image preprocessing ----------
+    def _letterbox(
+        self,
+        image: ndarray,
+        new_shape: tuple[int, int],
+        color=(114, 114, 114),
+    ) -> tuple[ndarray, float, tuple[float, float]]:
+        h, w = image.shape[:2]
+        new_w, new_h = new_shape
+        ratio = min(new_w / w, new_h / h)
+        resized_w = int(round(w * ratio))
+        resized_h = int(round(h * ratio))
+        if (resized_w, resized_h) != (w, h):
+            interp = cv2.INTER_CUBIC if ratio > 1.0 else cv2.INTER_LINEAR
+            image = cv2.resize(image, (resized_w, resized_h), interpolation=interp)
+        dw = (new_w - resized_w) / 2.0
+        dh = (new_h - resized_h) / 2.0
+        left = int(round(dw - 0.1))
+        right = int(round(dw + 0.1))
+        top = int(round(dh - 0.1))
+        bottom = int(round(dh + 0.1))
+        padded = cv2.copyMakeBorder(
+            image, top, bottom, left, right,
+            borderType=cv2.BORDER_CONSTANT, value=color,
+        )
+        return padded, ratio, (dw, dh)
+
+    def _preprocess(self, image: ndarray):
+        img, ratio, pad = self._letterbox(image, (self.input_width, self.input_height))
+        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0
+        img = np.transpose(img, (2, 0, 1))[None, ...]
+        return np.ascontiguousarray(img, dtype=self.input_dtype), ratio, pad
+
+    @staticmethod
+    def _clip_boxes(boxes: np.ndarray, image_size: tuple[int, int]) -> np.ndarray:
+        w, h = image_size
+        boxes[:, 0] = np.clip(boxes[:, 0], 0, w - 1)
+        boxes[:, 1] = np.clip(boxes[:, 1], 0, h - 1)
+        boxes[:, 2] = np.clip(boxes[:, 2], 0, w - 1)
+        boxes[:, 3] = np.clip(boxes[:, 3], 0, h - 1)
+        return boxes
+
+    # ---------- NMS primitives ----------
+    @staticmethod
+    def _hard_nms(boxes: np.ndarray, scores: np.ndarray, iou_thresh: float) -> np.ndarray:
+        N = len(boxes)
+        if N == 0:
+            return np.array([], dtype=np.intp)
+        boxes = np.asarray(boxes, dtype=np.float32)
+        scores = np.asarray(scores, dtype=np.float32)
+        order = np.argsort(-scores)
+        keep: list[int] = []
+        while len(order):
+            i = int(order[0])
+            keep.append(i)
+            if len(order) == 1:
+                break
+            rest = order[1:]
+            xx1 = np.maximum(boxes[i, 0], boxes[rest, 0])
+            yy1 = np.maximum(boxes[i, 1], boxes[rest, 1])
+            xx2 = np.minimum(boxes[i, 2], boxes[rest, 2])
+            yy2 = np.minimum(boxes[i, 3], boxes[rest, 3])
+            inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
+            area_i = (boxes[i, 2] - boxes[i, 0]) * (boxes[i, 3] - boxes[i, 1])
+            area_r = (boxes[rest, 2] - boxes[rest, 0]) * (boxes[rest, 3] - boxes[rest, 1])
+            iou = inter / (area_i + area_r - inter + 1e-7)
+            order = rest[iou <= iou_thresh]
+        return np.array(keep, dtype=np.intp)
+
+    def _soft_nms(
+        self,
+        boxes: np.ndarray,
+        scores: np.ndarray,
+        sigma: float,
+        score_thresh: float = 0.01,
+    ) -> tuple[np.ndarray, np.ndarray]:
+        N = len(boxes)
+        if N == 0:
+            return np.array([], dtype=np.intp), np.array([], dtype=np.float32)
+        boxes = boxes.astype(np.float32, copy=True)
+        scores = scores.astype(np.float32, copy=True)
+        order = np.arange(N)
+        for i in range(N):
+            max_pos = i + int(np.argmax(scores[i:]))
+            boxes[[i, max_pos]] = boxes[[max_pos, i]]
+            scores[[i, max_pos]] = scores[[max_pos, i]]
+            order[[i, max_pos]] = order[[max_pos, i]]
+            if i + 1 >= N:
+                break
+            xx1 = np.maximum(boxes[i, 0], boxes[i + 1:, 0])
+            yy1 = np.maximum(boxes[i, 1], boxes[i + 1:, 1])
+            xx2 = np.minimum(boxes[i, 2], boxes[i + 1:, 2])
+            yy2 = np.minimum(boxes[i, 3], boxes[i + 1:, 3])
+            inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
+            area_i = float(
+                (boxes[i, 2] - boxes[i, 0]) * (boxes[i, 3] - boxes[i, 1])
+            )
+            areas_j = (
+                np.maximum(0.0, boxes[i + 1:, 2] - boxes[i + 1:, 0])
+                * np.maximum(0.0, boxes[i + 1:, 3] - boxes[i + 1:, 1])
+            )
+            iou = inter / (area_i + areas_j - inter + 1e-7)
+            scores[i + 1:] *= np.exp(-(iou ** 2) / sigma)
+        mask = scores > score_thresh
+        return order[mask], scores[mask]
+
+    # ---------- raw-dets helper ----------
+    def _raw_dets(self, image: ndarray, conf: float) -> np.ndarray:
+        """Run a single forward pass and return [N, 5] dets in ORIGINAL image coords."""
+        x, ratio, (dw, dh) = self._preprocess(image)
+        out = self.session.run(self.output_names, {self.input_name: x})[0]
+        if out.ndim == 3:
+            out = out[0]
+        if out.shape[1] < 5:
+            return np.zeros((0, 5), dtype=np.float32)
+        boxes = out[:, :4].astype(np.float32)
+        scores = out[:, 4].astype(np.float32)
+        keep = scores >= conf
+        boxes, scores = boxes[keep], scores[keep]
+        if len(boxes) == 0:
+            return np.zeros((0, 5), dtype=np.float32)
+        boxes[:, [0, 2]] -= dw
+        boxes[:, [1, 3]] -= dh
+        boxes /= ratio
+        oh, ow = image.shape[:2]
+        boxes = self._clip_boxes(boxes, (ow, oh))
+        return np.concatenate([boxes, scores[:, None]], axis=1)
+
+    # ---------- primary pass: soft-NMS + hflip TTA ----------
+    def _primary(self, image: ndarray) -> np.ndarray:
+        d1 = self._raw_dets(image, self.conf_thres)
+        if self.use_tta:
+            flipped = cv2.flip(image, 1)
+            d2 = self._raw_dets(flipped, self.conf_thres)
+            if len(d2):
+                w = image.shape[1]
+                x1 = w - d2[:, 2]
+                x2 = w - d2[:, 0]
+                d2 = np.stack([x1, d2[:, 1], x2, d2[:, 3], d2[:, 4]], axis=1)
+            all_d = np.concatenate([d1, d2], axis=0) if len(d2) else d1
+        else:
+            all_d = d1
+        if len(all_d) == 0:
+            return np.zeros((0, 5), dtype=np.float32)
+        # soft-NMS, then hard-NMS
+        keep_idx, scores = self._soft_nms(all_d[:, :4].copy(), all_d[:, 4].copy(), sigma=self.sigma)
+        if len(keep_idx) == 0:
+            return np.zeros((0, 5), dtype=np.float32)
+        merged = np.concatenate([all_d[keep_idx, :4], scores[:, None]], axis=1)
+        keep = self._hard_nms(merged[:, :4], merged[:, 4], self.iou_thres)
+        merged = merged[keep]
+        if len(merged) > self.max_det:
+            merged = merged[np.argsort(-merged[:, 4])[: self.max_det]]
+        return merged
+
+    # ---------- single-image predict ----------
+    def _predict_single(self, image: ndarray) -> list[BoundingBox]:
+        if image is None or not isinstance(image, np.ndarray) or image.ndim != 3:
             return []
-        dets.sort(key=lambda x: -x[0])
-        keep = []
-        used = [False] * len(dets)
-        for i in range(len(dets)):
-            if used[i]:
-                continue
-            keep.append(dets[i])
-            for j in range(i + 1, len(dets)):
-                if used[j]:
-                    continue
-                # compute IoU
-                ax1, ay1, ax2, ay2 = dets[i][1], dets[i][2], dets[i][3], dets[i][4]
-                bx1, by1, bx2, by2 = dets[j][1], dets[j][2], dets[j][3], dets[j][4]
-                ix1 = max(ax1, bx1); iy1 = max(ay1, by1)
-                ix2 = min(ax2, bx2); iy2 = min(ay2, by2)
-                inter = max(0, ix2-ix1) * max(0, iy2-iy1)
-                aa = (ax2-ax1)*(ay2-ay1); bb = (bx2-bx1)*(by2-by1)
-                iou = inter / (aa + bb - inter + 1e-6)
-                if iou > self.iou_threshold:
-                    used[j] = True
-        return keep
-
-    def _infer_single(self, image_bgr: ndarray) -> list[BoundingBox]:
-        orig_h, orig_w = image_bgr.shape[:2]
-        all_dets = []
-
-        # Full image pass
-        all_dets.extend(self._run_single(image_bgr))
-
-        # 2x2 tile passes
-        tw = orig_w // 2
-        th = orig_h // 2
-        ox = int(tw * self.tile_overlap)
-        oy = int(th * self.tile_overlap)
-        tiles = [
-            (0, 0, tw + ox, th + oy),
-            (tw - ox, 0, orig_w, th + oy),
-            (0, th - oy, tw + ox, orig_h),
-            (tw - ox, th - oy, orig_w, orig_h),
-        ]
-        for tx1, ty1, tx2, ty2 in tiles:
-            tx1 = max(0, tx1); ty1 = max(0, ty1)
-            tx2 = min(orig_w, tx2); ty2 = min(orig_h, ty2)
-            crop = image_bgr[ty1:ty2, tx1:tx2]
-            tile_dets = self._run_single(crop)
-            for conf, x1, y1, x2, y2 in tile_dets:
-                all_dets.append((conf, x1 + tx1, y1 + ty1, x2 + tx1, y2 + ty1))
-
-        # NMS to merge overlapping detections
-        all_dets = self._nms(all_dets)
-
-        out_boxes = []
-        for conf, x1, y1, x2, y2 in all_dets:
-            bx1 = max(0, min(orig_w, math.floor(x1)))
-            by1 = max(0, min(orig_h, math.floor(y1)))
-            bx2 = max(0, min(orig_w, math.ceil(x2)))
-            by2 = max(0, min(orig_h, math.ceil(y2)))
-            bw = bx2 - bx1
-            bh = by2 - by1
-            if bw < 6 or bh < 6 or bw * bh < 80:
-                continue
-            if max(bw / max(bh, 1), bh / max(bw, 1)) > 10:
+        if image.shape[0] <= 0 or image.shape[1] <= 0 or image.shape[2] != 3:
+            return []
+        if image.dtype != np.uint8:
+            image = image.astype(np.uint8)
+
+        dets = self._primary(image)
+
+        results: list[BoundingBox] = []
+        for row in dets:
+            x1, y1, x2, y2, conf = row.tolist()
+            if x2 <= x1 or y2 <= y1:
                 continue
-            out_boxes.append(
+            results.append(
                 BoundingBox(
-                    x1=bx1, y1=by1, x2=bx2, y2=by2,
+                    x1=int(math.floor(x1)),
+                    y1=int(math.floor(y1)),
+                    x2=int(math.ceil(x2)),
+                    y2=int(math.ceil(y2)),
                     cls_id=0,
-                    conf=max(0.0, min(1.0, conf)),
+                    conf=float(conf),
                 )
             )
-        return out_boxes
+        return results
 
+    # ---------- chute entrypoint ----------
     def predict_batch(
-        self, batch_images: list[ndarray], offset: int, n_keypoints: int,
+        self,
+        batch_images: list[ndarray],
+        offset: int,
+        n_keypoints: int,
     ) -> list[TVFrameResult]:
-        results = []
-        for idx, image in enumerate(batch_images):
-            boxes = self._infer_single(image)
-            keypoints = [(0, 0) for _ in range(max(0, int(n_keypoints)))]
-            results.append(TVFrameResult(
-                frame_id=offset + idx, boxes=boxes, keypoints=keypoints))
+        results: list[TVFrameResult] = []
+        for frame_number_in_batch, image in enumerate(batch_images):
+            try:
+                boxes = self._predict_single(image)
+            except Exception as e:
+                print(f"Inference failed for frame {offset + frame_number_in_batch}: {e}")
+                boxes = []
+            results.append(
+                TVFrameResult(
+                    frame_id=offset + frame_number_in_batch,
+                    boxes=boxes,
+                    keypoints=[(0, 0) for _ in range(max(0, int(n_keypoints)))],
+                )
+            )
         return results