feat(frontend): improve UI for detection and chat interaction

app.py

@@ -55,9 +55,11 @@ from jobs.storage import (
     get_job_storage,
     get_output_video_path,
 )
-from utils.gpt_reasoning import estimate_threat_gpt, encode_frame_to_b64
+from utils.gpt_reasoning import estimate_threat_gpt
 from utils.threat_chat import chat_about_threats
-from utils.relevance import evaluate_relevance, evaluate_relevance_llm
+from utils.relevance import evaluate_relevance
+from utils.enrichment import run_enrichment
+from utils.schemas import AssessmentStatus
 from utils.mission_parser import parse_mission_text, build_broad_queries, MissionParseError
 
 logging.basicConfig(level=logging.INFO)
@@ -87,78 +89,41 @@ async def _enrich_first_frame_gpt(
     if not enable_gpt or not detections:
         return
     try:
-        # LLM relevance filter (if LLM_EXTRACTED mode)
-        gpt_dets = detections
-        if mission_spec and mission_spec.parse_mode == "LLM_EXTRACTED":
-            unique_labels = list({
-                d.get("label", "").lower()
-                for d in detections if d.get("label")
-            })
-            relevant_labels = await asyncio.to_thread(
-                evaluate_relevance_llm, unique_labels, mission_spec.operator_text
-            )
-            mission_spec.relevance_criteria.required_classes = list(relevant_labels)
-            # Apply deterministic filter with refined classes
+        # Non-LLM_EXTRACTED relevance filter runs BEFORE run_enrichment (FAST_PATH case)
+        if mission_spec and mission_spec.parse_mode != "LLM_EXTRACTED":
             for d in detections:
                 decision = evaluate_relevance(d, mission_spec.relevance_criteria)
                 d["mission_relevant"] = decision.relevant
                 d["relevance_reason"] = decision.reason
-            gpt_dets = [d for d in detections if d.get("mission_relevant", True)]
-        elif mission_spec:
-            for d in detections:
-                decision = evaluate_relevance(d, mission_spec.relevance_criteria)
-                d["mission_relevant"] = decision.relevant
-                d["relevance_reason"] = decision.reason
-            gpt_dets = [d for d in detections if d.get("mission_relevant", True)]
+            filtered = [d for d in detections if d.get("mission_relevant", True)]
+            if not filtered:
+                for det in detections:
+                    det["assessment_status"] = AssessmentStatus.ASSESSED
+                get_job_storage().update(job_id, first_frame_detections=detections)
+                logging.info("All detections non-relevant for job %s; marked ASSESSED", job_id)
+                return
 
-        if not gpt_dets:
-            # All detections filtered as not relevant — mark ASSESSED and persist
-            for det in detections:
-                det["assessment_status"] = "ASSESSED"
-            storage = get_job_storage()
-            storage.update(
-                job_id,
-                first_frame_detections=detections,
-            )
-            logging.info("All detections non-relevant for job %s; marked ASSESSED", job_id)
-            return
-
-        # GPT threat assessment
-        frame_b64 = encode_frame_to_b64(frame)
         gpt_results = await asyncio.to_thread(
-            estimate_threat_gpt,
-            detections=gpt_dets,
-            mission_spec=mission_spec,
-            image_b64=frame_b64,
+            run_enrichment, 0, frame, detections, mission_spec,
+            job_id=job_id,
         )
         logging.info("Background GPT enrichment complete for job %s", job_id)
 
-        # Merge GPT results into detections using the same object IDs used in
-        # the GPT request payload.
-        gpt_obj_ids = []
-        for i, det in enumerate(gpt_dets):
-            obj_id = det.get("track_id") or det.get("id")
-            if obj_id is None:
-                obj_id = f"T{str(i + 1).zfill(2)}"
-            gpt_obj_ids.append(str(obj_id))
-
-        for det, obj_id in zip(gpt_dets, gpt_obj_ids):
-            if obj_id in gpt_results:
-                info = gpt_results[obj_id]
-                det.update(info)
-                det["gpt_raw"] = info
-                det.setdefault("assessment_frame_index", 0)
-                det["assessment_status"] = info.get("assessment_status", "ASSESSED")
-            else:
-                det.setdefault("assessment_status", "UNASSESSED")
+        if not gpt_results:
+            # All detections filtered as not relevant
+            for det in detections:
+                det["assessment_status"] = AssessmentStatus.ASSESSED
+            get_job_storage().update(job_id, first_frame_detections=detections)
+            logging.info("All detections non-relevant for job %s; marked ASSESSED", job_id)
+            return
 
+        # Tag any remaining detections without an assessment status
         for det in detections:
             if "assessment_status" not in det:
-                det["assessment_status"] = "UNASSESSED"
+                det["assessment_status"] = AssessmentStatus.UNASSESSED
 
         # Update stored job so frontend polls pick up GPT data
-        storage = get_job_storage()
-        storage.update(
+        get_job_storage().update(
             job_id,
             first_frame_detections=detections,
             first_frame_gpt_results=gpt_results,
@@ -549,12 +514,14 @@ async def detect_async_endpoint(
     asyncio.create_task(process_video_async(job_id))
 
     # Fire-and-forget: enrich first-frame detections with GPT in background.
-    # Segmentation mode already runs one-shot GPT in GSAM2 writer, so skip this
-    # path there to avoid duplicate GPT calls on frame 0.
-    if mode != "segmentation":
-        asyncio.create_task(_enrich_first_frame_gpt(
-            job_id, processed_frame, detections, enable_gpt, mission_spec,
-        ))
+    # Runs for ALL modes including segmentation — first-frame detections from
+    # process_first_frame() already have stable track IDs (T01, T02, ...) and
+    # valid bboxes, so there's no reason to defer.  The GSAM2 writer's
+    # enrichment thread will see the cached results via first_frame_gpt_results
+    # in JobStorage and skip the duplicate call on frame 0.
+    asyncio.create_task(_enrich_first_frame_gpt(
+        job_id, processed_frame, detections, enable_gpt, mission_spec,
+    ))
 
     response_data = {
         "job_id": job_id,

frontend/index.html

@@ -282,6 +282,7 @@
   <script src="./js/core/video.js"></script>
   <script src="./js/core/hel.js"></script>
   <script src="./js/ui/logging.js"></script>
+  <script src="./js/core/gptMapping.js"></script>
   <script src="./js/core/tracker.js"></script>
   <script src="./js/api/client.js"></script>
   <script src="./js/ui/overlays.js"></script>

frontend/js/api/client.js

@@ -126,26 +126,7 @@ APP.api.client._syncGptFromDetections = function (rawDets, logLabel) {
             const existing = (state.detections || []).find(d => d.id === tid);
             if (existing && rd.gpt_raw) {
                 const g = rd.gpt_raw;
-                const rangeStr = g.range_estimate && g.range_estimate !== "Unknown"
-                    ? g.range_estimate + " (est.)" : "Unknown";
-                existing.features = {
-                    "Type": g.object_type || "Unknown",
-                    "Size": g.size || "Unknown",
-                    "Threat Lvl": (g.threat_level || g.threat_level_score || "?") + "/10",
-                    "Status": g.threat_classification || "?",
-                    "Weapons": (g.visible_weapons || []).join(", ") || "None Visible",
-                    "Readiness": g.weapon_readiness || "Unknown",
-                    "Motion": g.motion_status || "Unknown",
-                    "Range": rangeStr,
-                    "Bearing": g.bearing || "Unknown",
-                    "Intent": g.tactical_intent || "Unknown",
-                };
-                const dynFeats = g.dynamic_features || [];
-                for (const feat of dynFeats) {
-                    if (feat && feat.key && feat.value) {
-                        existing.features[feat.key] = feat.value;
-                    }
-                }
+                existing.features = APP.core.gptMapping.buildFeatures(g);
                 existing.threat_level_score = rd.threat_level_score || g.threat_level_score || 0;
                 existing.threat_classification = rd.threat_classification || g.threat_classification || "Unknown";
                 existing.weapon_readiness = rd.weapon_readiness || g.weapon_readiness || "Unknown";

frontend/js/core/gptMapping.js

@@ -0,0 +1,46 @@
+/**
+ * gptMapping.js — canonical GPT-raw → features field mapping.
+ *
+ * Replaces 4 identical inline mapping blocks across main.js, client.js,
+ * and tracker.js (2 locations).
+ */
+APP.core.gptMapping = {};
+
+/** Frozen assessment-status string constants. */
+APP.core.gptMapping.STATUS = Object.freeze({
+    ASSESSED: "ASSESSED",
+    UNASSESSED: "UNASSESSED",
+    STALE: "STALE",
+    PENDING_GPT: "PENDING_GPT",
+});
+
+/**
+ * Build a features object from a gpt_raw payload.
+ *
+ * @param {Object|null|undefined} gptRaw - The gpt_raw dict from a detection.
+ * @returns {Object} Features key-value map (empty object if gptRaw is falsy).
+ */
+APP.core.gptMapping.buildFeatures = function (gptRaw) {
+    if (!gptRaw) return {};
+    const rangeStr = gptRaw.range_estimate && gptRaw.range_estimate !== "Unknown"
+        ? gptRaw.range_estimate + " (est.)" : "Unknown";
+    const features = {
+        "Type": gptRaw.object_type || "Unknown",
+        "Size": gptRaw.size || "Unknown",
+        "Threat Lvl": (gptRaw.threat_level || gptRaw.threat_level_score || "?") + "/10",
+        "Status": gptRaw.threat_classification || "?",
+        "Weapons": (gptRaw.visible_weapons || []).join(", ") || "None Visible",
+        "Readiness": gptRaw.weapon_readiness || "Unknown",
+        "Motion": gptRaw.motion_status || "Unknown",
+        "Range": rangeStr,
+        "Bearing": gptRaw.bearing || "Unknown",
+        "Intent": gptRaw.tactical_intent || "Unknown",
+    };
+    const dynFeats = gptRaw.dynamic_features || [];
+    for (const feat of dynFeats) {
+        if (feat && feat.key && feat.value) {
+            features[feat.key] = feat.value;
+        }
+    }
+    return features;
+};

frontend/js/core/tracker.js

@@ -206,29 +206,11 @@ APP.core.tracker.syncWithBackend = async function (frameIdx) {
                 // Mission relevance and assessment status
                 mission_relevant: d.mission_relevant ?? null,
                 relevance_reason: d.relevance_reason || null,
-                assessment_status: d.assessment_status || "UNASSESSED",
+                assessment_status: d.assessment_status || APP.core.gptMapping.STATUS.UNASSESSED,
                 assessment_frame_index: d.assessment_frame_index ?? null,
                 // GPT raw data for feature table
                 gpt_raw: d.gpt_raw || null,
-                features: d.gpt_raw ? (() => {
-                    const f = {
-                        "Type": d.gpt_raw.object_type || "Unknown",
-                        "Size": d.gpt_raw.size || "Unknown",
-                        "Threat Lvl": (d.gpt_raw.threat_level || d.gpt_raw.threat_level_score || "?") + "/10",
-                        "Status": d.gpt_raw.threat_classification || "?",
-                        "Weapons": (d.gpt_raw.visible_weapons || []).join(", ") || "None Visible",
-                        "Readiness": d.gpt_raw.weapon_readiness || "Unknown",
-                        "Motion": d.gpt_raw.motion_status || "Unknown",
-                        "Range": d.gpt_raw.range_estimate && d.gpt_raw.range_estimate !== "Unknown"
-                            ? d.gpt_raw.range_estimate + " (est.)" : "Unknown",
-                        "Bearing": d.gpt_raw.bearing || "Unknown",
-                        "Intent": d.gpt_raw.tactical_intent || "Unknown",
-                    };
-                    for (const feat of (d.gpt_raw.dynamic_features || [])) {
-                        if (feat && feat.key && feat.value) f[feat.key] = feat.value;
-                    }
-                    return f;
-                })() : {},
+                features: APP.core.gptMapping.buildFeatures(d.gpt_raw),
                 // Keep UI state fields
                 lastSeen: Date.now(),
                 state: "TRACK"
@@ -248,7 +230,7 @@ APP.core.tracker.syncWithBackend = async function (frameIdx) {
                 if (!cached || track.gpt_raw) continue;
                 const g = cached.gpt_raw;
                 track.gpt_raw = g;
-                track.assessment_status = cached.assessment_status || "ASSESSED";
+                track.assessment_status = cached.assessment_status || APP.core.gptMapping.STATUS.ASSESSED;
                 track.threat_level_score = cached.threat_level_score || g.threat_level_score || 0;
                 track.threat_classification = cached.threat_classification || g.threat_classification || "Unknown";
                 track.weapon_readiness = cached.weapon_readiness || g.weapon_readiness || "Unknown";
@@ -256,22 +238,7 @@ APP.core.tracker.syncWithBackend = async function (frameIdx) {
                 track.gpt_direction = cached.gpt_direction || null;
                 track.mission_relevant = cached.mission_relevant ?? track.mission_relevant;
                 track.relevance_reason = cached.relevance_reason || track.relevance_reason;
-                track.features = {
-                    "Type": g.object_type || "Unknown",
-                    "Size": g.size || "Unknown",
-                    "Threat Lvl": (g.threat_level || g.threat_level_score || "?") + "/10",
-                    "Status": g.threat_classification || "?",
-                    "Weapons": (g.visible_weapons || []).join(", ") || "None Visible",
-                    "Readiness": g.weapon_readiness || "Unknown",
-                    "Motion": g.motion_status || "Unknown",
-                    "Range": g.range_estimate && g.range_estimate !== "Unknown"
-                        ? g.range_estimate + " (est.)" : "Unknown",
-                    "Bearing": g.bearing || "Unknown",
-                    "Intent": g.tactical_intent || "Unknown",
-                };
-                for (const feat of (g.dynamic_features || [])) {
-                    if (feat && feat.key && feat.value) track.features[feat.key] = feat.value;
-                }
+                track.features = APP.core.gptMapping.buildFeatures(g);
             }
         }
 

frontend/js/main.js

@@ -522,34 +522,8 @@ document.addEventListener("DOMContentLoaded", () => {
                 ? { x: d.bbox[0], y: d.bbox[1], w: d.bbox[2] - d.bbox[0], h: d.bbox[3] - d.bbox[1] }
                 : { x: 0, y: 0, w: 10, h: 10 };
 
-            // Range display: qualify GPT-estimated distances (INV-10)
-            const rangeDisplay = d.gpt_raw && d.gpt_raw.range_estimate && d.gpt_raw.range_estimate !== "Unknown"
-                ? d.gpt_raw.range_estimate + " (est.)"
-                : "Unknown";
-
-            // Build features from universal schema
-            let features = {};
-            if (d.gpt_raw) {
-                features = {
-                    "Type": d.gpt_raw.object_type || "Unknown",
-                    "Size": d.gpt_raw.size || "Unknown",
-                    "Threat Lvl": (d.gpt_raw.threat_level || d.gpt_raw.threat_level_score || "?") + "/10",
-                    "Status": d.gpt_raw.threat_classification || "?",
-                    "Weapons": (d.gpt_raw.visible_weapons || []).join(", ") || "None Visible",
-                    "Readiness": d.gpt_raw.weapon_readiness || "Unknown",
-                    "Motion": d.gpt_raw.motion_status || "Unknown",
-                    "Range": rangeDisplay,
-                    "Bearing": d.gpt_raw.bearing || "Unknown",
-                    "Intent": d.gpt_raw.tactical_intent || "Unknown",
-                };
-                // Spread dynamic features as extra key-value pairs
-                const dynFeats = d.gpt_raw.dynamic_features || [];
-                for (const feat of dynFeats) {
-                    if (feat && feat.key && feat.value) {
-                        features[feat.key] = feat.value;
-                    }
-                }
-            }
+            // Build features from universal schema via canonical mapping
+            let features = APP.core.gptMapping.buildFeatures(d.gpt_raw);
 
             return {
                 id,
@@ -578,7 +552,7 @@ document.addEventListener("DOMContentLoaded", () => {
                 // Mission relevance and assessment status
                 mission_relevant: d.mission_relevant ?? null,
                 relevance_reason: d.relevance_reason || null,
-                assessment_status: d.assessment_status || "UNASSESSED",
+                assessment_status: d.assessment_status || APP.core.gptMapping.STATUS.UNASSESSED,
                 assessment_frame_index: d.assessment_frame_index ?? null,
             };
         });

frontend/js/ui/cards.js

@@ -27,7 +27,8 @@ APP.ui.cards.renderFrameTrackList = function () {
     }
 
     // Sort: ASSESSED first (by threat score), then UNASSESSED, then STALE
-    const statusOrder = { "ASSESSED": 0, "UNASSESSED": 1, "STALE": 2 };
+    const S = APP.core.gptMapping.STATUS;
+    const statusOrder = { [S.ASSESSED]: 0, [S.UNASSESSED]: 1, [S.STALE]: 2 };
     const sorted = [...dets].sort((a, b) => {
         const statusA = statusOrder[a.assessment_status] ?? 1;
         const statusB = statusOrder[b.assessment_status] ?? 1;
@@ -68,14 +69,14 @@ APP.ui.cards.renderFrameTrackList = function () {
 
         // Assessment status badge
         let statusBadge = "";
-        const assessStatus = det.assessment_status || "UNASSESSED";
-        if (assessStatus === "UNASSESSED") {
+        const assessStatus = det.assessment_status || S.UNASSESSED;
+        if (assessStatus === S.UNASSESSED) {
             statusBadge = '<span class="badgemini" style="background:#6c757d; color:white">UNASSESSED</span>';
-        } else if (assessStatus === "STALE") {
+        } else if (assessStatus === S.STALE) {
             statusBadge = '<span class="badgemini" style="background:#ffc107; color:#333">STALE</span>';
         } else if (det.threat_level_score > 0) {
             statusBadge = `<span class="badgemini" style="background:${det.threat_level_score >= 8 ? '#ff4d4d' : '#ff9f43'}; color:white">T-${det.threat_level_score}</span>`;
-        } else if (assessStatus === "ASSESSED") {
+        } else if (assessStatus === S.ASSESSED) {
             statusBadge = '<span class="badgemini" style="background:#17a2b8; color:white">ASSESSED</span>';
         }
 

utils/threat_chat.py

@@ -2,11 +2,12 @@
 Threat Chat Module - GPT-powered Q&A about detected threats.
 """
 
-import os
-import json
 import logging
 from typing import List, Dict, Any
 
+from utils.openai_client import chat_completion, extract_content, get_api_key, OpenAIAPIError
+from utils.gpt_reasoning import _DOMAIN_ROLES
+
 logger = logging.getLogger(__name__)
 
 
@@ -26,11 +27,7 @@ def chat_about_threats(
     Returns:
         GPT's response as a string.
     """
-    import urllib.request
-    import urllib.error
-
-    api_key = os.environ.get("OPENAI_API_KEY")
-    if not api_key:
+    if not get_api_key():
         logger.warning("OPENAI_API_KEY not set. Cannot process threat chat.")
         return "Error: OpenAI API key not configured."
 
@@ -42,17 +39,9 @@ def chat_about_threats(
 
     # Domain-aware role selection
     domain = "GENERIC"
-    role_label = "Tactical Intelligence Officer"
     if mission_spec_dict:
         domain = mission_spec_dict.get("domain", "GENERIC")
-        if domain == "NAVAL":
-            role_label = "Naval Tactical Intelligence Officer"
-        elif domain == "GROUND":
-            role_label = "Ground Surveillance Intelligence Officer"
-        elif domain == "AERIAL":
-            role_label = "Air Surveillance Intelligence Officer"
-        elif domain == "URBAN":
-            role_label = "Urban Surveillance Intelligence Officer"
+    role_label = _DOMAIN_ROLES.get(domain, _DOMAIN_ROLES["GENERIC"])
 
     # Build mission context block (INV-8: mission context forwarded to LLM calls)
     mission_block = ""
@@ -89,29 +78,16 @@ def chat_about_threats(
         "temperature": 0.3,
     }
 
-    headers = {
-        "Content-Type": "application/json",
-        "Authorization": f"Bearer {api_key}"
-    }
-
     try:
-        req = urllib.request.Request(
-            "https://api.openai.com/v1/chat/completions",
-            data=json.dumps(payload).encode('utf-8'),
-            headers=headers,
-            method="POST"
-        )
-        with urllib.request.urlopen(req, timeout=30) as response:
-            resp_data = json.loads(response.read().decode('utf-8'))
-            
-        content = resp_data['choices'][0]['message'].get('content', '')
+        resp_data = chat_completion(payload)
+        content, _refusal = extract_content(resp_data)
         return content.strip() if content else "No response generated."
-        
-    except urllib.error.HTTPError as e:
-        logger.error(f"OpenAI API HTTP error: {e.code} - {e.reason}")
-        return f"API Error: {e.reason}"
+
+    except OpenAIAPIError as e:
+        logger.error("OpenAI API error: %s", e)
+        return f"API Error: {e}"
     except Exception as e:
-        logger.error(f"Threat chat failed: {e}")
+        logger.error("Threat chat failed: %s", e)
         return f"Error processing question: {str(e)}"