refactor(gsam2): make SAM2 detector-agnostic

app.py

@@ -248,7 +248,7 @@ async def detect_endpoint(
     mode: str = Form(...),
     queries: str = Form(""),
     detector: str = Form("hf_yolov8"),
-    segmenter: str = Form("gsam2_large"),
+    segmenter: str = Form("GSAM2-L"),
     enable_depth: bool = Form(False),
     enable_gpt: bool = Form(True),
 ):
@@ -260,7 +260,7 @@ async def detect_endpoint(
         mode: Detection mode (object_detection, segmentation, drone_detection)
         queries: Comma-separated object classes for object_detection mode
         detector: Model to use (hf_yolov8, detr_resnet50, grounding_dino)
-        segmenter: Segmentation model to use (gsam2_small, gsam2_base, gsam2_large)
+        segmenter: Segmentation model to use (GSAM2-S, GSAM2-B, GSAM2-L)
         enable_depth: Whether to run legacy depth estimation (default: False)
         drone_detection uses the dedicated drone_yolo model.
 
@@ -302,6 +302,7 @@ async def detect_endpoint(
                 output_path,
                 query_list,
                 segmenter_name=segmenter,
+                detector_name="grounding_dino",
                 num_maskmem=7,
             )
         except ValueError as exc:
@@ -402,7 +403,7 @@ async def detect_async_endpoint(
     mode: str = Form(...),
     queries: str = Form(""),
     detector: str = Form("hf_yolov8"),
-    segmenter: str = Form("gsam2_large"),
+    segmenter: str = Form("GSAM2-L"),
     depth_estimator: str = Form("depth"),
     depth_scale: float = Form(25.0),
     enable_depth: bool = Form(False),
@@ -491,7 +492,6 @@ async def detect_async_endpoint(
         )
         cv2.imwrite(str(first_frame_path), processed_frame)
         # GPT and depth are now handled in the async pipeline (enrichment thread)
-        depth_map = None
         first_frame_gpt_results = None
     except Exception:
         logging.exception("First-frame processing failed.")
@@ -910,7 +910,7 @@ async def chat_threat_endpoint(
 async def benchmark_endpoint(
     video: UploadFile = File(...),
     queries: str = Form("person,car,truck"),
-    segmenter: str = Form("gsam2_large"),
+    segmenter: str = Form("GSAM2-L"),
     step: int = Form(60),
     num_maskmem: Optional[int] = Form(None),
 ):
@@ -1036,7 +1036,7 @@ async def benchmark_profile(
     video: UploadFile = File(...),
     mode: str = Form("detection"),
     detector: str = Form("hf_yolov8"),
-    segmenter: str = Form("gsam2_large"),
+    segmenter: str = Form("GSAM2-L"),
     queries: str = Form("person,car,truck"),
     max_frames: int = Form(100),
     warmup_frames: int = Form(5),
@@ -1102,7 +1102,7 @@ async def benchmark_analysis(
     video: UploadFile = File(...),
     mode: str = Form("detection"),
     detector: str = Form("hf_yolov8"),
-    segmenter: str = Form("gsam2_large"),
+    segmenter: str = Form("GSAM2-L"),
     queries: str = Form("person,car,truck"),
     max_frames: int = Form(100),
     warmup_frames: int = Form(5),

frontend/index.html

@@ -75,9 +75,9 @@
                   <option value="grounding_dino" data-kind="object">Large</option>
                 </optgroup>
                 <optgroup label="Segmentation Models">
-                  <option value="gsam2_large" data-kind="segmentation">SAM2 Large</option>
-                  <option value="gsam2_base" data-kind="segmentation">SAM2 Base+</option>
-                  <option value="gsam2_small" data-kind="segmentation">SAM2 Small</option>
+                  <option value="GSAM2-L" data-kind="segmentation">GSAM2-L</option>
+                  <option value="GSAM2-B" data-kind="segmentation">GSAM2-B</option>
+                  <option value="GSAM2-S" data-kind="segmentation">GSAM2-S</option>
                 </optgroup>
                 <optgroup label="Drone Detection Models">
                   <option value="drone_yolo" data-kind="drone">Drone</option>

frontend/js/main.js

@@ -363,11 +363,11 @@ document.addEventListener("DOMContentLoaded", () => {
             } else if (kind === "drone") {
                 mode = "drone_detection";
                 detectorParam = selectedValue;
-                segmenterParam = "gsam2_large";
+                segmenterParam = "GSAM2-L";
             } else {
                 mode = "object_detection";
                 detectorParam = selectedValue;
-                segmenterParam = "gsam2_large";
+                segmenterParam = "GSAM2-L";
             }
 
             const form = new FormData();

inference.py

@@ -1631,6 +1631,7 @@ def run_grounded_sam2_tracking(
     _perf_metrics: Optional[Dict[str, float]] = None,
     _perf_lock=None,
     num_maskmem: Optional[int] = None,
+    detector_name: Optional[str] = None,
 ) -> str:
     """Run Grounded-SAM-2 video tracking pipeline.
 
@@ -1645,7 +1646,7 @@ def run_grounded_sam2_tracking(
     from utils.video import extract_frames_to_jpeg_dir
     from models.segmenters.grounded_sam2 import MaskDictionary, ObjectInfo, LazyFrameObjects
 
-    active_segmenter = segmenter_name or "gsam2_large"
+    active_segmenter = segmenter_name or "GSAM2-L"
     logging.info(
         "Grounded-SAM-2 tracking: segmenter=%s, queries=%s, step=%d",
         active_segmenter, queries, step,
@@ -2120,6 +2121,8 @@ def run_grounded_sam2_tracking(
                 # ---------- Single-GPU fallback ----------
                 device_str = "cuda:0" if torch.cuda.is_available() else "cpu"
                 _seg_kw = {"num_maskmem": num_maskmem} if num_maskmem is not None else {}
+                if detector_name is not None:
+                    _seg_kw["detector_name"] = detector_name
 
                 if _perf_metrics is not None:
                     _t_load = time.perf_counter()
@@ -2176,6 +2179,8 @@ def run_grounded_sam2_tracking(
                 segmenters = []
                 with ThreadPoolExecutor(max_workers=num_gpus) as pool:
                     _seg_kw_multi = {"num_maskmem": num_maskmem} if num_maskmem is not None else {}
+                    if detector_name is not None:
+                        _seg_kw_multi["detector_name"] = detector_name
                     futs = [
                         pool.submit(
                             load_segmenter_on_device,

jobs/background.py

@@ -2,12 +2,10 @@ import asyncio
 import logging
 from datetime import datetime
 
-import torch
-
 from jobs.models import JobStatus
-from jobs.storage import get_job_storage, get_depth_output_path, get_first_frame_depth_path
+from jobs.storage import get_job_storage
 from jobs.streaming import create_stream, remove_stream
-from inference import run_inference, run_grounded_sam2_tracking, run_depth_inference
+from inference import run_inference, run_grounded_sam2_tracking
 
 
 async def process_video_async(job_id: str) -> None:
@@ -41,6 +39,7 @@ async def process_video_async(job_id: str) -> None:
                 mission_spec=job.mission_spec,
                 first_frame_gpt_results=job.first_frame_gpt_results,
                 num_maskmem=7,
+                detector_name=job.detector_name,
             )
         else:
             detections_list = None

models/segmenters/grounded_sam2.py

@@ -1,6 +1,6 @@
 """Grounded-SAM-2 segmenter with continuous-ID video tracking.
 
-Combines Grounding DINO (open-vocabulary detection) with SAM2's video
+Combines an object detector (open-vocabulary or closed-set) with SAM2's video
 predictor to produce temporally consistent segmentation masks with
 persistent object IDs across an entire video.
 
@@ -13,7 +13,7 @@ import logging
 import time
 from contextlib import nullcontext
 from dataclasses import dataclass, field
-from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, TYPE_CHECKING
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 import numpy as np
 import torch
@@ -308,15 +308,26 @@ _SAM2_HF_MODELS = {
 }
 
 
+def _det_label_names(det) -> List[str]:
+    """Extract string labels from a DetectionResult, with fallback."""
+    num_boxes = len(det.boxes) if det.boxes is not None else 0
+    if det.label_names is not None and len(det.label_names) > 0:
+        return list(det.label_names)
+    if det.labels is not None and len(det.labels) > 0:
+        return [str(l) for l in det.labels]
+    return ["object"] * num_boxes
+
+
 # ---------------------------------------------------------------------------
 # Grounded-SAM-2 Segmenter
 # ---------------------------------------------------------------------------
 
 class GroundedSAM2Segmenter(Segmenter):
-    """SAM2 video segmenter driven by Grounding DINO detections.
+    """SAM2 video segmenter driven by an injected object detector.
 
-    For single-frame mode (``predict``), uses GDINO + SAM2 image predictor.
-    For video mode (``process_video``), uses GDINO on keyframes + SAM2 video
+    Any ``ObjectDetector`` can be used (defaults to Grounding DINO).
+    For single-frame mode (``predict``), uses detector + SAM2 image predictor.
+    For video mode (``process_video``), uses detector on keyframes + SAM2 video
     predictor for temporal mask propagation with continuous object IDs.
     """
 
@@ -330,12 +341,15 @@ class GroundedSAM2Segmenter(Segmenter):
         step: int = 20,
         iou_threshold: float = 0.5,
         num_maskmem: Optional[int] = None,
+        detector_name: Optional[str] = None,
     ):
         self.model_size = model_size
         self.step = step
         self.iou_threshold = iou_threshold
         self.num_maskmem = num_maskmem  # None = use default (7)
-        self.name = f"gsam2_{model_size}"
+        self._detector_name = detector_name  # None = "grounding_dino"
+        _size_suffix = {"small": "S", "base": "B", "large": "L"}
+        self.name = f"GSAM2-{_size_suffix[model_size]}"
 
         if device:
             self.device = device
@@ -345,7 +359,7 @@ class GroundedSAM2Segmenter(Segmenter):
         # Lazy-loaded model handles
         self._video_predictor = None
         self._image_predictor = None
-        self._gdino_detector = None
+        self._detector = None
         self._models_loaded = False
 
     # -- Lazy loading -------------------------------------------------------
@@ -388,10 +402,11 @@ class GroundedSAM2Segmenter(Segmenter):
             self._patch_num_maskmem(self._video_predictor, self.num_maskmem)
             logging.info("Patched video predictor num_maskmem → %d", self.num_maskmem)
 
-        # Reuse existing Grounding DINO detector from our codebase
-        from models.detectors.grounding_dino import GroundingDinoDetector
+        # Load detector by name (defaults to Grounding DINO)
+        from models.model_loader import load_detector_on_device
 
-        self._gdino_detector = GroundingDinoDetector(device=self.device)
+        det_name = self._detector_name or "grounding_dino"
+        self._detector = load_detector_on_device(det_name, self.device)
 
         self._models_loaded = True
         logging.info("Grounded-SAM-2 models loaded successfully.")
@@ -476,13 +491,13 @@ class GroundedSAM2Segmenter(Segmenter):
     def predict(
         self, frame: np.ndarray, text_prompts: Optional[list] = None
     ) -> SegmentationResult:
-        """Run GDINO + SAM2 image predictor on a single frame."""
+        """Run detector + SAM2 image predictor on a single frame."""
         self._ensure_models_loaded()
 
         prompts = text_prompts or ["object"]
 
-        # Run Grounding DINO to get boxes
-        det = self._gdino_detector.predict(frame, prompts)
+        # Run detector to get boxes
+        det = self._detector.predict(frame, prompts)
         if det.boxes is None or len(det.boxes) == 0:
             return SegmentationResult(
                 masks=np.zeros((0, frame.shape[0], frame.shape[1]), dtype=bool),
@@ -539,11 +554,11 @@ class GroundedSAM2Segmenter(Segmenter):
         image: "Image",
         text_prompts: List[str],
     ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor], List[str]]:
-        """Run GDINO + SAM2 image predictor on a single keyframe.
+        """Run detector + SAM2 image predictor on a single keyframe.
 
         Args:
             image: PIL Image in RGB mode.
-            text_prompts: Text queries for Grounding DINO.
+            text_prompts: Text queries for the detector.
 
         Returns:
             ``(masks, boxes, labels)`` where *masks* is an ``(N, H, W)``
@@ -554,26 +569,12 @@ class GroundedSAM2Segmenter(Segmenter):
         self._ensure_models_loaded()
         _pm = getattr(self, '_perf_metrics', None)
 
-        prompt = self._gdino_detector._build_prompt(text_prompts)
-        gdino_processor = self._gdino_detector.processor
-        gdino_model = self._gdino_detector.model
-
         if _pm is not None:
             _t0 = time.perf_counter()
 
-        inputs = gdino_processor(
-            images=image, text=prompt, return_tensors="pt"
-        )
-        inputs = {k: v.to(self.device) for k, v in inputs.items()}
-
-        with torch.no_grad():
-            outputs = gdino_model(**inputs)
-
-        results = self._gdino_detector._post_process(
-            outputs,
-            inputs["input_ids"],
-            target_sizes=[image.size[::-1]],
-        )
+        # Convert PIL RGB → numpy BGR for detector.predict()
+        frame_bgr = np.array(image)[:, :, ::-1].copy()
+        det = self._detector.predict(frame_bgr, text_prompts)
 
         if _pm is not None:
             _pl = getattr(self, '_perf_lock', None)
@@ -583,21 +584,18 @@ class GroundedSAM2Segmenter(Segmenter):
             else:
                 _pm["gdino_total_ms"] += _d
 
-        input_boxes = results[0]["boxes"]
-        det_labels = results[0].get("text_labels") or results[0].get("labels", [])
-        if torch.is_tensor(det_labels):
-            det_labels = det_labels.detach().cpu().tolist()
-        det_labels = [str(l) for l in det_labels]
-
-        if input_boxes.shape[0] == 0:
+        if det.boxes is None or len(det.boxes) == 0:
             return None, None, []
 
+        input_boxes = torch.tensor(det.boxes, device=self.device, dtype=torch.float32)
+        det_labels = _det_label_names(det)
+
         # SAM2 image predictor
         if _pm is not None:
             _t1 = time.perf_counter()
 
         self._image_predictor.set_image(np.array(image))
-        masks, scores = self._predict_masks_gpu(input_boxes)
+        masks, _ = self._predict_masks_gpu(input_boxes)
 
         if _pm is not None:
             _pl = getattr(self, '_perf_lock', None)
@@ -721,7 +719,7 @@ class GroundedSAM2Segmenter(Segmenter):
         Args:
             frame_dir: Directory containing JPEG frames.
             frame_names: Sorted list of frame filenames.
-            text_prompts: Text queries for Grounding DINO.
+            text_prompts: Text queries for the detector.
             on_segment: Optional callback invoked after each segment completes.
                 Receives ``{frame_idx: {obj_id: ObjectInfo}}`` for the segment.
 
@@ -735,11 +733,6 @@ class GroundedSAM2Segmenter(Segmenter):
 
         device = self.device
         step = self.step
-        prompt = self._gdino_detector._build_prompt(text_prompts)
-
-        # HF processor for Grounding DINO (reuse from our detector)
-        gdino_processor = self._gdino_detector.processor
-        gdino_model = self._gdino_detector.model
 
         total_frames = len(frame_names)
         logging.info(
@@ -783,24 +776,12 @@ class GroundedSAM2Segmenter(Segmenter):
 
                 mask_dict = MaskDictionary()
 
-                # -- Grounding DINO detection on keyframe --
+                # -- Detector on keyframe --
                 if _pm is not None:
                     _t_gd = time.perf_counter()
 
-                inputs = gdino_processor(
-                    images=image, text=prompt, return_tensors="pt"
-                )
-                inputs = {k: v.to(device) for k, v in inputs.items()}
-
-                with torch.no_grad():
-                    outputs = gdino_model(**inputs)
-
-                # Use GDINO detector's _post_process for transformers version compat
-                results = self._gdino_detector._post_process(
-                    outputs,
-                    inputs["input_ids"],
-                    target_sizes=[image.size[::-1]],
-                )
+                frame_bgr = np.array(image)[:, :, ::-1].copy()
+                det = self._detector.predict(frame_bgr, text_prompts)
 
                 if _pm is not None:
                     _pl = getattr(self, '_perf_lock', None)
@@ -810,13 +791,14 @@ class GroundedSAM2Segmenter(Segmenter):
                     else:
                         _pm["gdino_total_ms"] += _d
 
-                input_boxes = results[0]["boxes"]
-                det_labels = results[0].get("text_labels") or results[0].get("labels", [])
-                if torch.is_tensor(det_labels):
-                    det_labels = det_labels.detach().cpu().tolist()
-                det_labels = [str(l) for l in det_labels]
+                if det.boxes is None or len(det.boxes) == 0:
+                    input_boxes = torch.zeros((0, 4), device=device)
+                    det_labels = []
+                else:
+                    input_boxes = torch.tensor(det.boxes, device=device, dtype=torch.float32)
+                    det_labels = _det_label_names(det)
 
-                if input_boxes.shape[0] == 0:
+                if len(input_boxes) == 0:
                     logging.info("No detections on keyframe %d, propagating previous masks", start_idx)
                     # Fill empty results for this segment
                     seg_results: Dict[int, Dict[int, ObjectInfo]] = {}
@@ -842,7 +824,7 @@ class GroundedSAM2Segmenter(Segmenter):
                     _t_si = time.perf_counter()
 
                 self._image_predictor.set_image(np.array(image))
-                masks, scores = self._predict_masks_gpu(input_boxes)
+                masks, _ = self._predict_masks_gpu(input_boxes)
 
                 if _pm is not None:
                     _pl = getattr(self, '_perf_lock', None)

models/segmenters/model_loader.py

@@ -5,12 +5,12 @@ from typing import Callable, Dict, Optional
 from .base import Segmenter
 from .grounded_sam2 import GroundedSAM2Segmenter
 
-DEFAULT_SEGMENTER = "gsam2_large"
+DEFAULT_SEGMENTER = "GSAM2-L"
 
 _REGISTRY: Dict[str, Callable[..., Segmenter]] = {
-    "gsam2_small": lambda **kw: GroundedSAM2Segmenter(model_size="small", **kw),
-    "gsam2_base": lambda **kw: GroundedSAM2Segmenter(model_size="base", **kw),
-    "gsam2_large": lambda **kw: GroundedSAM2Segmenter(model_size="large", **kw),
+    "GSAM2-S": lambda **kw: GroundedSAM2Segmenter(model_size="small", **kw),
+    "GSAM2-B": lambda **kw: GroundedSAM2Segmenter(model_size="base", **kw),
+    "GSAM2-L": lambda **kw: GroundedSAM2Segmenter(model_size="large", **kw),
 }
 
 
@@ -37,7 +37,7 @@ def load_segmenter(name: Optional[str] = None) -> Segmenter:
     Load a segmenter by name.
 
     Args:
-        name: Segmenter name (default: gsam2_large)
+        name: Segmenter name (default: GSAM2-L)
 
     Returns:
         Cached segmenter instance

utils/roofline.py

@@ -21,9 +21,9 @@ _MODEL_FLOPS: Dict[str, float] = {
     "drone_yolo": 78.9,          # Same arch as YOLOv8m
 
     # Segmentation models (GFLOPs per keyframe)
-    "gsam2_small": 48.0,         # SAM2 small encoder
-    "gsam2_base": 96.0,          # SAM2 base encoder
-    "gsam2_large": 200.0,        # SAM2 large encoder
+    "GSAM2-S": 48.0,             # SAM2 small encoder
+    "GSAM2-B": 96.0,             # SAM2 base encoder
+    "GSAM2-L": 200.0,            # SAM2 large encoder
     "gsam2_tiny": 24.0,          # SAM2 tiny encoder
 }
 
@@ -34,9 +34,9 @@ _MODEL_BYTES: Dict[str, float] = {
     "detr_resnet50": 166.0,
     "grounding_dino": 340.0,
     "drone_yolo": 52.0,
-    "gsam2_small": 92.0,
-    "gsam2_base": 180.0,
-    "gsam2_large": 400.0,
+    "GSAM2-S": 92.0,
+    "GSAM2-B": 180.0,
+    "GSAM2-L": 400.0,
     "gsam2_tiny": 46.0,
 }