detector.py

"""Lazy-loaded FastSAM wrapper for click-to-annotate.

The user clicks a pixel on the active page; we run FastSAM's
"segment-everything" pass over the image, then pick the smallest mask whose
binary pixel value at the click is above threshold (smallest = most local
object, the one the user most likely meant).

The model file (~74 MB for FastSAM-s) auto-downloads on first use.
"""
from __future__ import annotations

import base64
import io
import threading
from typing import Optional

import numpy as np
from PIL import Image

_MODEL_LOCK = threading.Lock()
_MODEL = None
_MODEL_NAME = "FastSAM-s.pt"  # ~74 MB, fast enough for interactive use


def _get_model():
    global _MODEL
    if _MODEL is not None:
        return _MODEL
    with _MODEL_LOCK:
        if _MODEL is None:
            from ultralytics import FastSAM  # imported lazily to keep startup quick
            _MODEL = FastSAM(_MODEL_NAME)
    return _MODEL


def segment_at_point(
    image_b64: str, x: int, y: int, *,
    imgsz: int = 1024, conf: float = 0.4, iou: float = 0.9,
) -> Optional[dict]:
    """Return ``{"bbox_px": [x1, y1, x2, y2]}`` for the FastSAM mask under
    the click, or ``None`` if no mask covers that pixel.

    Coordinates are in the natural image frame (the same frame the
    frontend uses when sending the click).
    """
    raw = base64.b64decode(image_b64)
    img = Image.open(io.BytesIO(raw)).convert("RGB")
    arr = np.array(img)
    img_h, img_w = arr.shape[:2]

    if not (0 <= x < img_w and 0 <= y < img_h):
        return None

    model = _get_model()
    results = model.predict(
        arr,
        device="cpu",
        retina_masks=True,
        imgsz=imgsz,
        conf=conf,
        iou=iou,
        verbose=False,
    )
    if not results:
        return None
    result = results[0]
    if result.masks is None or len(result.masks.data) == 0:
        return None

    masks = result.masks.data.cpu().numpy() if hasattr(result.masks.data, "cpu") else np.asarray(result.masks.data)
    # Masks are (N, H, W) floats in [0, 1] — may not be at the original resolution if
    # retina_masks is unavailable for this build; rescale the click accordingly.
    mh, mw = masks.shape[1], masks.shape[2]
    mx = int(round(x * mw / img_w))
    my = int(round(y * mh / img_h))
    mx = max(0, min(mw - 1, mx))
    my = max(0, min(mh - 1, my))

    candidates: list[tuple[int, list[int]]] = []
    for i in range(masks.shape[0]):
        if masks[i, my, mx] <= 0.5:
            continue
        ys, xs = np.where(masks[i] > 0.5)
        if xs.size == 0:
            continue
        # bbox in mask-coordinates, then scaled to image-coordinates
        x1 = int(round(xs.min() * img_w / mw))
        y1 = int(round(ys.min() * img_h / mh))
        x2 = int(round((xs.max() + 1) * img_w / mw))
        y2 = int(round((ys.max() + 1) * img_h / mh))
        x1 = max(0, min(img_w, x1)); y1 = max(0, min(img_h, y1))
        x2 = max(0, min(img_w, x2)); y2 = max(0, min(img_h, y2))
        if x2 <= x1 or y2 <= y1:
            continue
        area = int(xs.size)
        candidates.append((area, [x1, y1, x2, y2]))

    if not candidates:
        return None

    # The smallest containing mask is almost always the most local / most
    # precise object the user pointed at.
    candidates.sort(key=lambda t: t[0])
    return {"bbox_px": candidates[0][1]}