fix: correct benchmark metrics — remove double-counting and add missing timers

- Fix sam_video_total_ms double-counting in single-GPU process_video() path
(propagate_segment already accumulates, outer timer was adding it twice)
- Fix id_reconciliation_ms in multi-GPU path to measure actual IoU work
instead of wall-clock queue waits
- Add model_load_ms and init_state_ms metrics for both single/multi-GPU paths

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

app.py

@@ -880,6 +880,8 @@ async def benchmark_endpoint(
         metrics = {
             "end_to_end_ms": 0.0,
             "frame_extraction_ms": 0.0,
+            "model_load_ms": 0.0,
+            "init_state_ms": 0.0,
             "tracking_total_ms": 0.0,
             "gdino_total_ms": 0.0,
             "sam_image_total_ms": 0.0,

inference.py

@@ -2048,10 +2048,15 @@ def run_grounded_sam2_tracking(
                 # ---------- Single-GPU fallback ----------
                 device_str = "cuda:0" if torch.cuda.is_available() else "cpu"
                 _seg_kw = {"num_maskmem": num_maskmem} if num_maskmem is not None else {}
+
+                if _perf_metrics is not None:
+                    _t_load = time.perf_counter()
+
                 segmenter = load_segmenter_on_device(active_segmenter, device_str, **_seg_kw)
                 _check_cancellation(job_id)
 
                 if _perf_metrics is not None:
+                    _perf_metrics["model_load_ms"] = (time.perf_counter() - _t_load) * 1000.0
                     segmenter._perf_metrics = _perf_metrics
                     segmenter._perf_lock = None
 
@@ -2082,6 +2087,9 @@ def run_grounded_sam2_tracking(
                 )
 
                 # Phase 1: Load one segmenter per GPU (parallel)
+                if _perf_metrics is not None:
+                    _t_load = time.perf_counter()
+
                 segmenters = []
                 with ThreadPoolExecutor(max_workers=num_gpus) as pool:
                     _seg_kw_multi = {"num_maskmem": num_maskmem} if num_maskmem is not None else {}
@@ -2098,6 +2106,7 @@ def run_grounded_sam2_tracking(
                 logging.info("Loaded %d segmenters", len(segmenters))
 
                 if _perf_metrics is not None:
+                    _perf_metrics["model_load_ms"] = (time.perf_counter() - _t_load) * 1000.0
                     import threading as _th
                     _actual_lock = _perf_lock or _th.Lock()
                     for seg in segmenters:
@@ -2105,6 +2114,9 @@ def run_grounded_sam2_tracking(
                         seg._perf_lock = _actual_lock
 
                 # Phase 2: Init SAM2 models/state per GPU (parallel)
+                if _perf_metrics is not None:
+                    _t_init = time.perf_counter()
+
                 def _init_seg_state(seg):
                     seg._ensure_models_loaded()
                     return seg._video_predictor.init_state(
@@ -2118,6 +2130,7 @@ def run_grounded_sam2_tracking(
                     inference_states = [f.result() for f in futs]
 
                 if _perf_metrics is not None:
+                    _perf_metrics["init_state_ms"] = (time.perf_counter() - _t_init) * 1000.0
                     _t_track = time.perf_counter()
 
                 # Phase 3: Parallel segment processing (queue-based workers)
@@ -2226,8 +2239,7 @@ def run_grounded_sam2_tracking(
 
                 # Phase 4: Streaming reconciliation — process segments in order
                 # as they arrive, feeding render_in incrementally.
-                if _perf_metrics is not None:
-                    _t_recon = time.perf_counter()
+                _recon_accum_ms = 0.0
 
                 global_id_counter = 0
                 sam2_masks = MaskDictionary()
@@ -2287,6 +2299,9 @@ def run_grounded_sam2_tracking(
                             continue
 
                         # Normalize keyframe masks to CPU before cross-GPU IoU matching.
+                        if _perf_metrics is not None:
+                            _t_rc = time.perf_counter()
+
                         for info in mask_dict.labels.values():
                             info.mask = _mask_to_cpu(info.mask)
 
@@ -2300,6 +2315,8 @@ def run_grounded_sam2_tracking(
                         )
 
                         if not mask_dict.labels:
+                            if _perf_metrics is not None:
+                                _recon_accum_ms += (time.perf_counter() - _t_rc) * 1000.0
                             for fi in range(
                                 start_idx, min(start_idx + step, total_frames)
                             ):
@@ -2327,6 +2344,9 @@ def run_grounded_sam2_tracking(
                                 )
                             tracking_results[frame_idx] = remapped
 
+                        if _perf_metrics is not None:
+                            _recon_accum_ms += (time.perf_counter() - _t_rc) * 1000.0
+
                         # Update running tracker with last frame of this segment
                         if segment_results:
                             last_fi = max(segment_results.keys())
@@ -2354,7 +2374,7 @@ def run_grounded_sam2_tracking(
                     t.join()
 
                 if _perf_metrics is not None:
-                    _perf_metrics["id_reconciliation_ms"] = (time.perf_counter() - _t_recon) * 1000.0
+                    _perf_metrics["id_reconciliation_ms"] = _recon_accum_ms
                     _perf_metrics["tracking_total_ms"] = (time.perf_counter() - _t_track) * 1000.0
 
                 logging.info(

models/segmenters/grounded_sam2.py

@@ -717,12 +717,23 @@ class GroundedSAM2Segmenter(Segmenter):
 
         with autocast_ctx:
             # Init SAM2 video predictor state
+            if _pm is not None:
+                _t_init = time.perf_counter()
+
             inference_state = self._video_predictor.init_state(
                 video_path=frame_dir,
                 offload_video_to_cpu=True,
                 async_loading_frames=True,
             )
 
+            if _pm is not None:
+                _pl = getattr(self, '_perf_lock', None)
+                _d = (time.perf_counter() - _t_init) * 1000.0
+                if _pl:
+                    with _pl: _pm["init_state_ms"] += _d
+                else:
+                    _pm["init_state_ms"] += _d
+
             for start_idx in range(0, total_frames, step):
                 logging.info("Processing keyframe %d / %d", start_idx, total_frames)
 
@@ -847,9 +858,8 @@ class GroundedSAM2Segmenter(Segmenter):
                     continue
 
                 # -- SAM2 video predictor: propagate masks --
-                if _pm is not None:
-                    _t_sv = time.perf_counter()
-
+                # NOTE: propagate_segment() already accumulates into
+                # _pm["sam_video_total_ms"], so no outer timer here.
                 segment_output = self.propagate_segment(
                     inference_state, start_idx, mask_dict, step,
                 )
@@ -870,14 +880,6 @@ class GroundedSAM2Segmenter(Segmenter):
                                 sam2_masks.mask_height = first_info.mask.shape[-2] if first_info.mask.ndim >= 2 else 0
                                 sam2_masks.mask_width = first_info.mask.shape[-1] if first_info.mask.ndim >= 2 else 0
 
-                if _pm is not None:
-                    _pl = getattr(self, '_perf_lock', None)
-                    _d = (time.perf_counter() - _t_sv) * 1000.0
-                    if _pl:
-                        with _pl: _pm["sam_video_total_ms"] += _d
-                    else:
-                        _pm["sam_video_total_ms"] += _d
-
         logging.info(
             "Grounded-SAM-2 tracking complete: %d frames, %d tracked objects",
             len(all_results), objects_count,