fix: track cards update during playback + periodic GPT re-analysis

Fix 4 interconnected bugs preventing track card updates during video
playback: normal tracking code trapped inside demo-mode guard,
renderFrameTrackList never called in animation loop, state.detections/
tracks disconnect, and asyncJobId nulled after completion blocking
syncWithBackend. Cards now update every 40 frames with GPT re-analysis
firing async on each cycle. New /detect/analyze-frame endpoint provides
per-frame GPT threat assessment.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

app.py

@@ -599,6 +599,59 @@ async def get_frame_tracks(job_id: str, frame_idx: int):
     return data or []
 
 
+@app.post("/detect/analyze-frame")
+async def analyze_frame(
+    image: UploadFile = File(...),
+    detections: str = Form(...),
+    job_id: str = Form(None),
+):
+    """Run GPT threat assessment on a single video frame."""
+    import json as json_module
+    from utils.gpt_reasoning import encode_frame_to_b64
+
+    dets = json_module.loads(detections)
+
+    # Look up mission_spec from stored job (if available)
+    mission_spec = None
+    if job_id:
+        job = get_job_storage().get(job_id)
+        if job:
+            mission_spec = job.mission_spec
+
+    # Decode uploaded image
+    image_bytes = await image.read()
+    nparr = np.frombuffer(image_bytes, np.uint8)
+    frame = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
+    if frame is None:
+        raise HTTPException(status_code=400, detail="Invalid image")
+
+    # Run GPT in thread pool (blocking OpenAI API call)
+    frame_b64 = encode_frame_to_b64(frame)
+    async with _GPT_SEMAPHORE:
+        gpt_results = await asyncio.to_thread(
+            estimate_threat_gpt,
+            detections=dets,
+            mission_spec=mission_spec,
+            image_b64=frame_b64,
+        )
+
+    # Merge GPT results into detection records
+    for d in dets:
+        oid = d.get("track_id") or d.get("id")
+        if oid and oid in gpt_results:
+            payload = gpt_results[oid]
+            d["gpt_raw"] = payload
+            d["assessment_status"] = payload.get("assessment_status", "ASSESSED")
+            d["threat_level_score"] = payload.get("threat_level_score", 0)
+            d["threat_classification"] = payload.get("threat_classification", "Unknown")
+            d["weapon_readiness"] = payload.get("weapon_readiness", "Unknown")
+            d["gpt_description"] = payload.get("gpt_description")
+            d["gpt_distance_m"] = payload.get("gpt_distance_m")
+            d["gpt_direction"] = payload.get("gpt_direction")
+
+    return dets
+
+
 @app.delete("/detect/job/{job_id}")
 async def cancel_job(job_id: str):
     """Cancel a running job."""

frontend/js/api/client.js

@@ -198,6 +198,7 @@ APP.api.client.pollAsyncJob = async function () {
                         await fetchDepthFirstFrame();
 
                         clearInterval(state.hf.asyncPollInterval);
+                        state.hf.completedJobId = state.hf.asyncJobId;  // preserve for post-completion sync
                         state.hf.asyncJobId = null;
                         setHfStatus("ready");
                         resolve();
@@ -306,6 +307,46 @@ APP.api.client.callHfObjectDetection = async function (canvas) {
     return await resp.json();
 };
 
+// Capture current video frame and send to backend for GPT analysis
+APP.api.client.analyzeFrame = async function (videoEl, tracks) {
+    const { state } = APP.core;
+    const { canvasToBlob } = APP.core.utils;
+
+    // Capture current video frame
+    const canvas = document.createElement("canvas");
+    canvas.width = videoEl.videoWidth;
+    canvas.height = videoEl.videoHeight;
+    canvas.getContext("2d").drawImage(videoEl, 0, 0);
+    const blob = await canvasToBlob(canvas);
+
+    // Convert normalized bbox (0-1) back to pixel coords for backend
+    const w = canvas.width, h = canvas.height;
+    const dets = tracks.map(t => ({
+        track_id: t.id,
+        label: t.label,
+        bbox: [
+            Math.round(t.bbox.x * w),
+            Math.round(t.bbox.y * h),
+            Math.round((t.bbox.x + t.bbox.w) * w),
+            Math.round((t.bbox.y + t.bbox.h) * h),
+        ],
+        score: t.score,
+    }));
+
+    const form = new FormData();
+    form.append("image", blob, "frame.jpg");
+    form.append("detections", JSON.stringify(dets));
+    const jobId = state.hf.asyncJobId || state.hf.completedJobId;
+    if (jobId) form.append("job_id", jobId);
+
+    const resp = await fetch(`${state.hf.baseUrl}/detect/analyze-frame`, {
+        method: "POST",
+        body: form,
+    });
+    if (!resp.ok) throw new Error(`Frame analysis failed: ${resp.statusText}`);
+    return await resp.json();
+};
+
 // Chat about threats using GPT
 APP.api.client.chatAboutThreats = async function (question, detections) {
     const { state } = APP.core;

frontend/js/core/state.js

@@ -14,6 +14,7 @@ APP.core.state = {
         baseUrl: (window.API_CONFIG?.BACKEND_BASE || window.API_CONFIG?.BASE_URL || "").replace(/\/$/, "") || window.location.origin,
         detector: "auto",
         asyncJobId: null,           // Current job ID from /detect/async
+        completedJobId: null,       // Preserved job ID for post-completion track sync
         asyncPollInterval: null,    // Polling timer handle
         firstFrameUrl: null,        // First frame preview URL
         firstFrameDetections: null, // First-frame detections from backend
@@ -56,7 +57,9 @@ APP.core.state = {
         selectedTrackId: null,
         beamOn: false,
         lastFrameTime: 0,
-        frameCount: 0
+        frameCount: 0,
+        _lastCardRenderFrame: 0,    // Frame count at last card render
+        _gptBusy: false             // Prevent overlapping GPT calls
     },
 
     frame: {

frontend/js/core/tracker.js

@@ -156,7 +156,7 @@ APP.core.tracker.matchAndUpdateTracks = function (dets, dtSec) {
 APP.core.tracker.syncWithBackend = async function (frameIdx) {
     const { state } = APP.core;
     const { $ } = APP.core.utils;
-    const jobId = state.hf.asyncJobId;
+    const jobId = state.hf.asyncJobId || state.hf.completedJobId;
 
     if (!jobId || !state.hf.baseUrl) return;
 
@@ -242,15 +242,18 @@ APP.core.tracker.syncWithBackend = async function (frameIdx) {
             }
         }
 
+        // Detect new objects before state update
+        const oldIds = new Set(state.tracker.tracks.map(t => t.id));
+        const brandNew = newTracks.filter(t => !oldIds.has(t.id));
+        if (brandNew.length > 0) {
+            state.tracker._newObjectDetected = true;
+            APP.ui.logging.log(`New objects: ${brandNew.map(t => t.id).join(", ")}`, "t");
+        }
+
         // Update state
         state.tracker.tracks = newTracks;
         state.detections = newTracks; // Keep synced
 
-        // Re-render track cards (same renderer as Tab 1)
-        if (APP.ui.cards && APP.ui.cards.renderFrameTrackList) {
-            APP.ui.cards.renderFrameTrackList();
-        }
-
     } catch (e) {
         console.warn("Track sync failed", e);
     }

frontend/js/core/video.js

@@ -118,6 +118,7 @@ APP.core.video.unloadVideo = async function (options = {}) {
     state.hf.busy = false;
     state.hf.lastError = null;
     state.hf.asyncJobId = null;
+    state.hf.completedJobId = null;
     state.hf.asyncStatus = "idle";
     state.hf.firstFrameUrl = null;
     state.hf.videoUrl = null;

frontend/js/main.js

@@ -620,6 +620,7 @@ document.addEventListener("DOMContentLoaded", () => {
         state.isReasoning = false;
         state.hf.busy = false;
         state.hf.asyncJobId = null;
+        state.hf.completedJobId = null;
         state.hf.asyncStatus = "cancelled";
 
         // Re-enable Reason button
@@ -684,9 +685,10 @@ document.addEventListener("DOMContentLoaded", () => {
 
         // Update tracker when engaged
         if (state.tracker.running && videoEngage && !videoEngage.paused) {
+            state.tracker.frameCount++;
 
-            // DEMO MODE BYPASS
             if (APP.core.demo.active && APP.core.demo.data) {
+                // DEMO MODE (keep existing demo track logic unchanged)
                 const demoTracks = getDemoFrameData(videoEngage.currentTime);
                 if (demoTracks) {
                     // Deep clone to avoid mutating source data
@@ -706,35 +708,62 @@ document.addEventListener("DOMContentLoaded", () => {
                     const h = videoEngage.videoHeight || state.frame.h || 720;
 
                     state.tracker.tracks.forEach(tr => {
-                        // Check if inputs are absolute pixels (if x > 1 or w > 1)
-                        // We assume demo data is in pixels (as per spec)
                         if (tr.bbox.x > 1 || tr.bbox.w > 1) {
                             tr.bbox.x /= w;
                             tr.bbox.y /= h;
                             tr.bbox.w /= w;
                             tr.bbox.h /= h;
                         }
-
-                        // Note: history in 'tr' is also in pixels in the source JSON.
-                        // But we don't normalize history here because radar.js currently handles raw pixels for history?
-                        // Actually, we should probably standardize everything to normalized if possible, 
-                        // but let's check radar.js first.
                     });
+                }
+            } else {
+                // ── NORMAL MODE ──
 
-                } else {
-                    // NORMAL MODE
-                    predictTracks(dt);
-
-                    // Sync with backend every few frames (approx 5Hz)
-                    if (t - state.tracker.lastHFSync > 200) {
-                        // Estimate frame index
-                        const fps = 30; // hardcoded for now, ideal: state.fps
-                        const frameIdx = Math.floor(videoEngage.currentTime * fps);
-                        // Only sync if we have a job ID
-                        if (state.hf.asyncJobId) {
-                            APP.core.tracker.syncWithBackend(frameIdx);
-                        }
-                        state.tracker.lastHFSync = t;
+                // (1) Every frame: smooth overlay animation
+                predictTracks(dt);
+
+                // (2) Every ~10 frames (333ms): backend sync for accurate positions
+                const jobId = state.hf.asyncJobId || state.hf.completedJobId;
+                if (jobId && (t - state.tracker.lastHFSync > 333)) {
+                    const frameIdx = Math.floor(videoEngage.currentTime * 30);
+                    APP.core.tracker.syncWithBackend(frameIdx);
+                    state.tracker.lastHFSync = t;
+                }
+
+                // (3) Every 40 frames OR new object: render cards + fire GPT
+                const framesSinceRender = state.tracker.frameCount - state.tracker._lastCardRenderFrame;
+                if (state.tracker._newObjectDetected || framesSinceRender >= 40) {
+                    // Immediate card render (current positions/labels, pre-GPT)
+                    renderFrameTrackList();
+                    state.tracker._lastCardRenderFrame = state.tracker.frameCount;
+                    state.tracker._newObjectDetected = false;
+
+                    // Fire async GPT analysis (non-blocking)
+                    if (!state.tracker._gptBusy && state.tracker.tracks.length > 0) {
+                        state.tracker._gptBusy = true;
+                        APP.api.client.analyzeFrame(videoEngage, state.tracker.tracks)
+                            .then(enriched => {
+                                // Merge GPT results into state.detections
+                                for (const rd of enriched) {
+                                    const tid = rd.track_id || rd.id;
+                                    const existing = (state.detections || []).find(d => d.id === tid);
+                                    if (existing && rd.gpt_raw) {
+                                        existing.gpt_raw = rd.gpt_raw;
+                                        existing.features = APP.core.gptMapping.buildFeatures(rd.gpt_raw);
+                                        existing.assessment_status = rd.assessment_status || "ASSESSED";
+                                        existing.threat_level_score = rd.threat_level_score || 0;
+                                        existing.gpt_description = rd.gpt_description || existing.gpt_description;
+                                        existing.gpt_distance_m = rd.gpt_distance_m || existing.gpt_distance_m;
+                                        existing.gpt_direction = rd.gpt_direction || existing.gpt_direction;
+                                    }
+                                }
+                                renderFrameTrackList();  // Re-render with GPT data
+                                state.tracker._gptBusy = false;
+                            })
+                            .catch(err => {
+                                console.warn("Frame GPT analysis failed:", err);
+                                state.tracker._gptBusy = false;
+                            });
                     }
                 }
             }