feat: mission-relevance coloring — red for relevant objects, gray for non-relevant

Backend-rendered video now uses GPT assessment verdicts to color bboxes
(detection mode) and masks (segmentation mode) red for mission-relevant
objects and gray for non-relevant. Unassessed tracks keep default colors.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

inference.py

@@ -21,7 +21,7 @@ from models.model_loader import load_detector, load_detector_on_device
 from models.segmenters.model_loader import load_segmenter, load_segmenter_on_device
 from models.depth_estimators.model_loader import load_depth_estimator, load_depth_estimator_on_device
 from utils.video import StreamingVideoWriter
-from jobs.storage import set_track_data, store_latest_frame
+from jobs.storage import set_track_data, store_latest_frame, get_job_storage
 from inspection.masks import rle_encode
 import tempfile
 import json as json_module
@@ -136,12 +136,37 @@ def _color_for_label(label: str) -> Tuple[int, int, int]:
     return (blue, green, red)
 
 
+# Mission-relevance colors (BGR)
+_COLOR_MISSION_RELEVANT = (0, 0, 255)      # red
+_COLOR_NOT_RELEVANT = (128, 128, 128)      # gray
+
+
+def _mission_colors_for_dets(
+    dets: list,
+    verdicts: Dict[str, bool],
+) -> Optional[List[Tuple[int, int, int]]]:
+    """Return per-detection BGR colors based on mission verdicts, or None if no verdicts."""
+    if not verdicts:
+        return None
+    colors = []
+    for d in dets:
+        tid = d.get("track_id")
+        if tid and tid in verdicts:
+            colors.append(_COLOR_MISSION_RELEVANT if verdicts[tid] else _COLOR_NOT_RELEVANT)
+        else:
+            # Unassessed — use default hash color
+            label = d.get("label", "obj")
+            colors.append(_color_for_label(label))
+    return colors
+
+
 def draw_boxes(
     frame: np.ndarray,
     boxes: np.ndarray,
     labels: Optional[Sequence[int]] = None,
     queries: Optional[Sequence[str]] = None,
     label_names: Optional[Sequence[str]] = None,
+    colors: Optional[Sequence[Tuple[int, int, int]]] = None,
 ) -> np.ndarray:
     output = frame.copy()
     if boxes is None:
@@ -158,7 +183,10 @@ def draw_boxes(
                 label = f"label_{label_idx}"
         else:
             label = f"label_{idx}"
-        color = (128, 128, 128) if not label else _color_for_label(label)
+        if colors is not None and idx < len(colors):
+            color = colors[idx]
+        else:
+            color = (128, 128, 128) if not label else _color_for_label(label)
         cv2.rectangle(output, (x1, y1), (x2, y2), color, thickness=2)
         if label:
             font = cv2.FONT_HERSHEY_SIMPLEX
@@ -190,6 +218,7 @@ def draw_masks(
     masks: np.ndarray,
     alpha: float = 0.65,
     labels: Optional[Sequence[str]] = None,
+    colors: Optional[Sequence[Tuple[int, int, int]]] = None,
 ) -> np.ndarray:
     output = frame.copy()
     if masks is None or len(masks) == 0:
@@ -206,9 +235,12 @@ def draw_masks(
         label = None
         if labels and idx < len(labels):
             label = labels[idx]
-        # Use a fallback key for consistent color even when no label text
-        color_key = label if label else f"object_{idx}"
-        color = _color_for_label(color_key)
+        if colors is not None and idx < len(colors):
+            color = colors[idx]
+        else:
+            # Use a fallback key for consistent color even when no label text
+            color_key = label if label else f"object_{idx}"
+            color = _color_for_label(color_key)
         overlay[mask_bool] = color
         output = cv2.addWeighted(output, 1.0, overlay, alpha, 0)
         contours, _ = cv2.findContours(
@@ -928,7 +960,12 @@ def run_inference(
                         if dets:
                             display_boxes = np.array([d['bbox'] for d in dets])
                             display_labels = [d.get('label', 'obj') for d in dets]
-                            p_frame = draw_boxes(p_frame, display_boxes, label_names=display_labels)
+                            # Apply mission-relevance colors (red/gray) if verdicts available
+                            det_colors = None
+                            if job_id:
+                                verdicts = get_job_storage().get_mission_verdicts(job_id)
+                                det_colors = _mission_colors_for_dets(dets, verdicts)
+                            p_frame = draw_boxes(p_frame, display_boxes, label_names=display_labels, colors=det_colors)
 
                         writer.write(p_frame)
 
@@ -1054,6 +1091,7 @@ def _gsam2_render_frame(
     height: int,
     width: int,
     frame_store=None,
+    mission_verdicts: Optional[Dict[str, bool]] = None,
 ) -> np.ndarray:
     """Render a single GSAM2 tracking frame (masks only, no bboxes). CPU-only."""
     if frame_store is not None:
@@ -1069,8 +1107,9 @@ def _gsam2_render_frame(
 
     masks_list: List[np.ndarray] = []
     mask_labels: List[str] = []
+    mask_colors: Optional[List[Tuple[int, int, int]]] = None if not mission_verdicts else []
 
-    for _obj_id, obj_info in frame_objects.items():
+    for obj_id, obj_info in frame_objects.items():
         mask = obj_info.mask
         label = obj_info.class_name
         if mask is not None:
@@ -1086,10 +1125,16 @@ def _gsam2_render_frame(
                 ).astype(bool)
             masks_list.append(mask_np)
             mask_labels.append(label)
+            if mask_colors is not None:
+                tid = str(obj_id)
+                if tid in mission_verdicts:
+                    mask_colors.append(_COLOR_MISSION_RELEVANT if mission_verdicts[tid] else _COLOR_NOT_RELEVANT)
+                else:
+                    mask_colors.append(_color_for_label(label or f"object_{obj_id}"))
 
     if masks_list:
         # Draw masks with labels — no bboxes for segmentation mode
-        frame = draw_masks(frame, np.stack(masks_list), labels=mask_labels)
+        frame = draw_masks(frame, np.stack(masks_list), labels=mask_labels, colors=mask_colors)
 
     return frame
 
@@ -1199,10 +1244,15 @@ def run_grounded_sam2_tracking(
                     if _perf_metrics is not None:
                         _t_r = time.perf_counter()
 
+                    # Fetch mission verdicts for coloring
+                    seg_verdicts = None
+                    if job_id:
+                        seg_verdicts = get_job_storage().get_mission_verdicts(job_id) or None
                     frm = _gsam2_render_frame(
                         frame_dir, frame_names, fidx, fobjs,
                         height, width,
                         frame_store=frame_store,
+                        mission_verdicts=seg_verdicts,
                     )
 
                     if _perf_metrics is not None:

jobs/storage.py

@@ -41,6 +41,7 @@ class JobStorage:
         self._tracks: Dict[str, Dict[int, list]] = {} # job_id -> {frame_idx -> tracks}
         self._latest_frames: Dict[str, any] = {}  # job_id -> np.ndarray
         self._mask_data: Dict[str, Dict[str, any]] = {}  # job_id -> {f"{frame_idx}:{track_id}" -> rle_dict}
+        self._mission_verdicts: Dict[str, Dict[str, bool]] = {}  # job_id -> {track_id -> mission_relevant}
         self._lock = RLock()
 
     def create(self, job: JobInfo) -> None:
@@ -93,6 +94,18 @@ class JobStorage:
             key = f"{frame_idx}:{track_id}"
             return self._mask_data.get(job_id, {}).get(key)
 
+    def set_mission_verdict(self, job_id: str, track_id: str, relevant: bool) -> None:
+        """Cache a mission-relevance verdict for a track."""
+        with self._lock:
+            if job_id not in self._mission_verdicts:
+                self._mission_verdicts[job_id] = {}
+            self._mission_verdicts[job_id][track_id] = relevant
+
+    def get_mission_verdicts(self, job_id: str) -> Dict[str, bool]:
+        """Return {track_id: mission_relevant} for all assessed tracks."""
+        with self._lock:
+            return dict(self._mission_verdicts.get(job_id, {}))
+
     def get_all_masks_for_frame(self, job_id: str, frame_idx: int) -> dict:
         """Return {track_id: rle_dict} for all objects in a frame."""
         with self._lock:
@@ -122,6 +135,7 @@ class JobStorage:
             self._tracks.pop(job_id, None)
             self._latest_frames.pop(job_id, None)
             self._mask_data.pop(job_id, None)
+            self._mission_verdicts.pop(job_id, None)
         shutil.rmtree(get_job_directory(job_id), ignore_errors=True)
 
     def cleanup_expired(self, max_age: timedelta) -> None:

models/isr/loop.py

@@ -117,6 +117,13 @@ async def run_isr_assessor_loop(
 def _merge_verdicts(storage, job_id: str, verdicts: dict, assessment_frame_idx: int) -> None:
     """Merge verdict data into all stored frames for matching track_ids."""
     with storage._lock:
+        # Update mission verdict cache for backend rendering
+        for tid, v in verdicts.items():
+            relevant = v.get("mission_relevant", True)
+            if job_id not in storage._mission_verdicts:
+                storage._mission_verdicts[job_id] = {}
+            storage._mission_verdicts[job_id][tid] = relevant
+
         frames = storage._tracks.get(job_id, {})
         for frame_idx, frame_tracks in frames.items():
             for det in frame_tracks: