feat: Add LLM post-processing relevance gate for broad detection

Move LLM's role from pre-processing (choosing detector classes) to
post-processing (filtering detected objects against mission intent).
Fast-path (comma-separated labels) is unchanged. LLM-path now detects
broadly (all COCO classes or domain-expanded queries), then asks GPT
once on frame 0 which labels are mission-relevant, caching the result
for deterministic filtering on all subsequent frames.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

app.py

@@ -57,7 +57,7 @@ from jobs.storage import (
 )
 from utils.gpt_reasoning import estimate_threat_gpt
 from utils.threat_chat import chat_about_threats
-from utils.mission_parser import parse_mission_text, MissionParseError
+from utils.mission_parser import parse_mission_text, build_broad_queries, MissionParseError
 
 logging.basicConfig(level=logging.INFO)
 
@@ -274,7 +274,7 @@ async def detect_endpoint(
     if queries.strip():
         try:
             mission_spec = parse_mission_text(queries.strip(), detector_name)
-            query_list = mission_spec.object_classes
+            query_list = build_broad_queries(detector_name, mission_spec)
         except MissionParseError as e:
             raise HTTPException(status_code=422, detail=str(e))
     else:
@@ -370,11 +370,12 @@ async def detect_async_endpoint(
     if queries.strip():
         try:
             mission_spec = parse_mission_text(queries.strip(), detector_name)
-            query_list = mission_spec.object_classes
+            query_list = build_broad_queries(detector_name, mission_spec)
             mission_mode = "MISSION"
             logging.info(
-                "Mission parsed: mode=%s classes=%s domain=%s(%s)",
-                mission_mode, query_list, mission_spec.domain, mission_spec.domain_source,
+                "Mission parsed: mode=%s classes=%s broad_queries=%s domain=%s(%s)",
+                mission_mode, mission_spec.object_classes, query_list,
+                mission_spec.domain, mission_spec.domain_source,
             )
         except MissionParseError as e:
             raise HTTPException(

inference.py

@@ -23,7 +23,7 @@ from models.depth_estimators.model_loader import load_depth_estimator, load_dept
 from models.depth_estimators.base import DepthEstimator
 from utils.video import extract_frames, write_video, VideoReader, VideoWriter, AsyncVideoReader
 from utils.gpt_reasoning import estimate_threat_gpt
-from utils.relevance import evaluate_relevance
+from utils.relevance import evaluate_relevance, evaluate_relevance_llm
 from jobs.storage import set_track_data
 import tempfile
 import json as json_module
@@ -732,25 +732,57 @@ def process_first_frame(
 
     # --- RELEVANCE GATE (between detection and GPT) ---
     if mission_spec:
-        relevant_dets = []
-        for det in detections:
-            decision = evaluate_relevance(det, mission_spec.relevance_criteria)
-            det["mission_relevant"] = decision.relevant
-            det["relevance_reason"] = decision.reason
-            if decision.relevant:
-                relevant_dets.append(det)
-            else:
-                logging.info(
-                    json_module.dumps({
-                        "event": "relevance_decision",
-                        "label": det.get("label"),
-                        "relevant": False,
-                        "reason": decision.reason,
-                        "required_classes": mission_spec.relevance_criteria.required_classes,
-                        "frame": 0,
-                    })
-                )
-        gpt_input_dets = relevant_dets
+        if mission_spec.parse_mode == "FAST_PATH":
+            # Deterministic gate (unchanged)
+            relevant_dets = []
+            for det in detections:
+                decision = evaluate_relevance(det, mission_spec.relevance_criteria)
+                det["mission_relevant"] = decision.relevant
+                det["relevance_reason"] = decision.reason
+                if decision.relevant:
+                    relevant_dets.append(det)
+                else:
+                    logging.info(
+                        json_module.dumps({
+                            "event": "relevance_decision",
+                            "label": det.get("label"),
+                            "relevant": False,
+                            "reason": decision.reason,
+                            "required_classes": mission_spec.relevance_criteria.required_classes,
+                            "frame": 0,
+                        })
+                    )
+            gpt_input_dets = relevant_dets
+        else:
+            # LLM_EXTRACTED: post-filter with GPT on frame 0
+            unique_labels = list({
+                d.get("label", "").lower()
+                for d in detections if d.get("label")
+            })
+            relevant_labels = evaluate_relevance_llm(
+                unique_labels, mission_spec.operator_text
+            )
+
+            # Cache GPT-approved labels into relevance_criteria for subsequent frames
+            mission_spec.relevance_criteria.required_classes = list(relevant_labels)
+
+            for det in detections:
+                label = (det.get("label") or "").lower()
+                is_relevant = label in relevant_labels
+                det["mission_relevant"] = is_relevant
+                det["relevance_reason"] = "ok" if is_relevant else "llm_excluded"
+                if not is_relevant:
+                    logging.info(
+                        json_module.dumps({
+                            "event": "relevance_decision",
+                            "label": det.get("label"),
+                            "relevant": False,
+                            "reason": "llm_excluded",
+                            "relevant_labels": list(relevant_labels),
+                            "frame": 0,
+                        })
+                    )
+            gpt_input_dets = [d for d in detections if d.get("mission_relevant")]
     else:
         # LEGACY mode: all detections pass, tagged as unresolved
         for det in detections:
@@ -1117,28 +1149,29 @@ def run_inference(
         # Initialize Tracker & Speed Estimator
         tracker = ByteTracker(frame_rate=fps)
         speed_est = SpeedEstimator(fps=fps)
-        
+        llm_filtered = False  # LLM post-filter runs once on frame 0
+
         try:
             with VideoWriter(output_video_path, fps, width, height) as writer:
                 while next_idx < total_frames:
                     # Fetch from queue
                     try:
                         while next_idx not in buffer:
-                            # Backpressure: If buffer gets too big due to out-of-order frames, 
-                            # we might want to warn or just hope for the best. 
+                            # Backpressure: If buffer gets too big due to out-of-order frames,
+                            # we might want to warn or just hope for the best.
                             # But here we are just consuming.
-                            
+
                             # However, if 'buffer' grows too large (because we are missing next_idx),
                             # we are effectively unbounded again if queue_out fills up with future frames.
                             # So we should monitor buffer size.
                             if len(buffer) > 200 and len(buffer) % 50 == 0:
                                 logging.warning("Writer buffer large (%d items), waiting for frame %d (GPT Latency?)...", len(buffer), next_idx)
-                            
+
                             item = queue_out.get(timeout=1.0) # wait
-                            
+
                             idx, p_frame, dets = item
                             buffer[idx] = (p_frame, dets)
-                            
+
                         # Write next_idx
                         p_frame, dets = buffer.pop(next_idx)
 
@@ -1147,7 +1180,27 @@ def run_inference(
                         dets = tracker.update(dets)
                         speed_est.estimate(dets)
 
-                        # --- RELEVANCE GATE ---
+                        # --- LLM POST-FILTER (frame 0 only, LLM_EXTRACTED mode) ---
+                        if (mission_spec
+                                and mission_spec.parse_mode == "LLM_EXTRACTED"
+                                and not llm_filtered
+                                and next_idx == 0):
+                            unique_labels = list({
+                                d.get("label", "").lower()
+                                for d in dets if d.get("label")
+                            })
+                            relevant_labels = evaluate_relevance_llm(
+                                unique_labels, mission_spec.operator_text
+                            )
+                            # Cache into relevance_criteria for all subsequent frames
+                            mission_spec.relevance_criteria.required_classes = list(relevant_labels)
+                            llm_filtered = True
+                            logging.info(
+                                "LLM post-filter applied on frame 0: relevant=%s",
+                                relevant_labels,
+                            )
+
+                        # --- RELEVANCE GATE (deterministic, uses updated criteria) ---
                         if mission_spec:
                             for d in dets:
                                 decision = evaluate_relevance(d, mission_spec.relevance_criteria)

utils/mission_parser.py

@@ -114,6 +114,7 @@ def _build_fast_path_spec(
         context_phrases=[],
         stripped_modifiers=[],
         operator_text=raw_text,
+        parse_mode="FAST_PATH",
         parse_confidence="HIGH",
         parse_warnings=warnings,
     )
@@ -328,11 +329,56 @@ def _validate_and_build(
         context_phrases=context_phrases,
         stripped_modifiers=stripped_modifiers,
         operator_text=raw_text,
+        parse_mode="LLM_EXTRACTED",
         parse_confidence=parse_confidence,
         parse_warnings=parse_warnings,
     )
 
 
+_DOMAIN_BROAD_CATEGORIES: dict[str, List[str]] = {
+    "NAVAL": ["vessel", "ship", "boat", "buoy", "person"],
+    "AERIAL": ["aircraft", "helicopter", "drone", "airplane"],
+    "GROUND": ["vehicle", "car", "truck", "person", "building"],
+    "URBAN": ["person", "vehicle", "car", "bicycle"],
+    "GENERIC": ["object"],
+}
+
+
+def build_broad_queries(
+    detector_key: str, mission_spec: MissionSpecification
+) -> List[str]:
+    """Build broad detector queries for LLM post-filter mode.
+
+    For FAST_PATH: return object_classes directly (unchanged behavior).
+    For COCO detectors (LLM_EXTRACTED): return ALL 80 COCO classes.
+    For open-vocab detectors (LLM_EXTRACTED): return LLM-extracted classes
+    PLUS broad domain categories to maximize recall.
+    """
+    if mission_spec.parse_mode == "FAST_PATH":
+        return mission_spec.object_classes
+
+    # LLM_EXTRACTED path: detect broadly
+    if _is_coco_only(detector_key):
+        # COCO detectors ignore queries anyway (DETR detects all 80;
+        # YOLOv8 falls back to all if no matches). Send everything.
+        return list(COCO_CLASSES)
+
+    # Open-vocab detector (e.g. Grounding DINO):
+    # Combine LLM-extracted classes with domain-specific broad categories
+    broad = list(mission_spec.object_classes)
+    domain_extras = _DOMAIN_BROAD_CATEGORIES.get(
+        mission_spec.domain, _DOMAIN_BROAD_CATEGORIES["GENERIC"]
+    )
+    seen = {c.lower() for c in broad}
+    for cat in domain_extras:
+        if cat.lower() not in seen:
+            broad.append(cat)
+            seen.add(cat.lower())
+
+    logger.info("Broad queries for %s: %s", detector_key, broad)
+    return broad
+
+
 def parse_mission_text(
     raw_text: str,
     detector_key: str,

utils/relevance.py

@@ -1,15 +1,21 @@
 """
 Object relevance evaluation — deterministic gate between detection and GPT assessment.
 
-Single public function: evaluate_relevance(detection, criteria) -> RelevanceDecision
+Public functions:
+  evaluate_relevance(detection, criteria) -> RelevanceDecision  (deterministic)
+  evaluate_relevance_llm(detected_labels, mission_text) -> set[str]  (LLM post-filter)
 
-INVARIANT INV-13 enforcement: This function accepts RelevanceCriteria, NOT
+INVARIANT INV-13 enforcement: evaluate_relevance() accepts RelevanceCriteria, NOT
 MissionSpecification. It cannot see context_phrases, stripped_modifiers, or any
 LLM-derived field. This is structural, not by convention.
 """
 
+import json
 import logging
-from typing import Any, Dict, NamedTuple
+import os
+import urllib.request
+import urllib.error
+from typing import Any, Dict, List, NamedTuple, Set
 
 from coco_classes import canonicalize_coco_name
 from utils.schemas import RelevanceCriteria
@@ -69,3 +75,82 @@ def evaluate_relevance(
                 return RelevanceDecision(True, "ok")
 
     return RelevanceDecision(False, "label_not_in_required_classes")
+
+
+def evaluate_relevance_llm(
+    detected_labels: List[str],
+    mission_text: str,
+) -> Set[str]:
+    """Ask GPT which detected labels are relevant to the mission.
+
+    Called ONCE on frame 0 with the unique labels found by the detector.
+    Returns a set of relevant label strings (lowercased).
+
+    On API failure, falls back to accepting all labels (fail-open, logged).
+    """
+    if not detected_labels:
+        return set()
+
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if not api_key:
+        logger.warning(
+            "OPENAI_API_KEY not set — LLM relevance filter falling back to accept-all"
+        )
+        return set(detected_labels)
+
+    prompt = (
+        f"Given this mission: \"{mission_text}\"\n\n"
+        f"Which of these detected object classes are relevant to the mission?\n"
+        f"{json.dumps(detected_labels)}\n\n"
+        "Return JSON: {\"relevant_labels\": [...]}\n"
+        "Only include labels from the provided list that are relevant to "
+        "accomplishing the mission. Be inclusive — if in doubt, include it."
+    )
+
+    payload = {
+        "model": "gpt-4o",
+        "temperature": 0.0,
+        "max_tokens": 200,
+        "response_format": {"type": "json_object"},
+        "messages": [
+            {"role": "system", "content": "You are a mission relevance filter. Return only JSON."},
+            {"role": "user", "content": prompt},
+        ],
+    }
+
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {api_key}",
+    }
+
+    try:
+        req = urllib.request.Request(
+            "https://api.openai.com/v1/chat/completions",
+            data=json.dumps(payload).encode("utf-8"),
+            headers=headers,
+            method="POST",
+        )
+        with urllib.request.urlopen(req, timeout=30) as response:
+            resp_data = json.loads(response.read().decode("utf-8"))
+
+        content = resp_data["choices"][0]["message"].get("content")
+        if not content:
+            logger.warning("GPT returned empty content for relevance filter — accept-all")
+            return set(detected_labels)
+
+        result = json.loads(content)
+        relevant = result.get("relevant_labels", detected_labels)
+        relevant_set = {label.lower() for label in relevant}
+
+        logger.info(
+            "LLM relevance filter: mission=%r detected=%s relevant=%s",
+            mission_text, detected_labels, relevant_set,
+        )
+        return relevant_set
+
+    except (urllib.error.HTTPError, urllib.error.URLError) as e:
+        logger.warning("LLM relevance API call failed: %s — accept-all fallback", e)
+        return set(detected_labels)
+    except (json.JSONDecodeError, KeyError, TypeError) as e:
+        logger.warning("LLM relevance response parse failed: %s — accept-all fallback", e)
+        return set(detected_labels)

utils/schemas.py

@@ -124,6 +124,13 @@ class MissionSpecification(BaseModel):
         description="Original unmodified mission text from the operator. Preserved for audit."
     )
 
+    # --- Parse mode ---
+    parse_mode: Literal["FAST_PATH", "LLM_EXTRACTED"] = Field(
+        default="FAST_PATH",
+        description="How this spec was created. FAST_PATH = comma-separated labels, "
+                    "LLM_EXTRACTED = natural language parsed by GPT."
+    )
+
     # --- LLM self-assessment ---
     parse_confidence: Literal["HIGH", "MEDIUM", "LOW"] = Field(
         ...,