Implement Fault Tolerance: UI polling timeout and backend FFmpeg detection

stitch_continuity_dashboard/code.html

@@ -102,6 +102,10 @@
                 <div class="size-1.5 rounded-full bg-green-500 animate-pulse"></div>
                 <span class="text-[10px] font-bold text-green-500 uppercase tracking-wider">Online</span>
             </div>
+            <div class="flex items-center gap-2 px-2 py-1 bg-green-500/10 border border-green-500/20 rounded-full ml-2">
+                <div class="size-1.5 rounded-full bg-green-500 animate-pulse"></div>
+                <span class="text-[10px] font-bold text-green-500 uppercase tracking-wider">Online</span>
+            </div>
         </div>
 
         <div class="flex items-center gap-3">
@@ -455,31 +459,45 @@
                 if (!res.ok) throw new Error(res.statusText);
                 const data = await res.json();
                 const poll = setInterval(async () => {
-                    const sRes = await fetch(`/status/${data.job_id}?t=${Date.now()}`);
-                    if (sRes.ok) {
-                        const s = await sRes.json();
-                        if (s.status === "completed") {
-                            clearInterval(poll);
-                            document.getElementById("bridge-content").innerHTML = `<video controls autoplay loop class="w-full h-full object-contain bg-black"><source src="${s.video_url}" type="video/mp4"></video>`;
-                            document.getElementById("bridge-card-outer").classList.replace("border-primary/20", "border-primary");
-                            document.getElementById("bridge-border").classList.replace("border-primary/50", "border-transparent");
-                            if (s.merged_video_url) {
-                                const dlBtn = document.getElementById("merged-download-btn");
-                                dlBtn.href = s.merged_video_url;
-                                document.getElementById("merged-download-container").classList.remove("hidden");
-                            }
-                            btn.innerHTML = "Done!";
-                            setTimeout(() => {
+                    failureCount = window.fc || 0;
+                    try {
+                        const sRes = await fetch(`/status/${data.job_id}?t=${Date.now()}`);
+                        if (sRes.ok) {
+                            window.fc = 0;
+                            const s = await sRes.json();
+                            if (s.status === "completed") {
+                                clearInterval(poll);
+                                document.getElementById("bridge-content").innerHTML = `<video controls autoplay loop class="w-full h-full object-contain bg-black"><source src="${s.video_url}" type="video/mp4"></video>`;
+                                document.getElementById("bridge-card-outer").classList.replace("border-primary/20", "border-primary");
+                                document.getElementById("bridge-border").classList.replace("border-primary/50", "border-transparent");
+                                if (s.merged_video_url) {
+                                    const dlBtn = document.getElementById("merged-download-btn");
+                                    dlBtn.href = s.merged_video_url;
+                                    document.getElementById("merged-download-container").classList.remove("hidden");
+                                }
+                                btn.innerHTML = "Done!";
+                                setTimeout(() => {
+                                    btn.disabled = false;
+                                    btn.innerHTML = `<span class="material-symbols-outlined">auto_fix_high</span> Generate Video`;
+                                }, 3000);
+                            } else if (s.status === "error") {
+                                clearInterval(poll);
+                                alert(s.log);
                                 btn.disabled = false;
-                                btn.innerHTML = `<span class="material-symbols-outlined">auto_fix_high</span> Generate Video`;
-                            }, 3000);
-                        } else if (s.status === "error") {
+                                btn.innerHTML = "Try Again";
+                            } else {
+                                btn.innerHTML = `${s.log} (${s.progress}%)`;
+                            }
+                        } else {
+                            throw new Error("Server Error");
+                        }
+                    } catch (e) {
+                        window.fc = (window.fc || 0) + 1;
+                        if (window.fc > 5) {
                             clearInterval(poll);
-                            alert(s.log);
+                            alert("Connection lost. The active job may get lost but you can try refreshing.");
                             btn.disabled = false;
-                            btn.innerHTML = "Try Again";
-                        } else {
-                            btn.innerHTML = `${s.log} (${s.progress}%)`;
+                            btn.innerHTML = "Connection Failed";
                         }
                     }
                 }, 1500);

utils.py

@@ -48,42 +48,49 @@ def save_video_bytes(bytes_data, suffix=".mp4") -> str:
     return f.name
 
 def normalize_video(input_path):
-    """Converts any video to a standard 1080p, 24fps, silent MP4 intermediate."""
+    """Helper to normalize video. Returns None if ffmpeg missing."""
+    if not shutil.which("ffmpeg"): return None
+
     output_path = input_path.replace(".mp4", "_norm.mp4")
     cmd = [
         "ffmpeg", "-y", "-i", input_path,
         "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,setsar=1,fps=24,format=yuv420p",
-        "-c:v", "libx264", "-preset", "ultrafast", "-crf", "28", "-an", # Remove audio to prevent mixing crashes
+        "-c:v", "libx264", "-preset", "ultrafast", "-crf", "28",
+        "-an",
         output_path
     ]
     subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
     return output_path
 
 def stitch_videos(path_a, path_b, path_c, output_path):
-    """ Robust Stitching: Normalizes clips individually first, then concats. This avoids the 'complex filter' crashes common with mismatched inputs. """
-    logger.info(f"🧵 Stitching: {path_a} + {path_b} + {path_c}")
+    """ Attempts to stitch videos. RETURNS: output_path if successful, NONE if ffmpeg is missing/fails. """
+    # 1. CHECK IF FFMPEG EXISTS
+    if not shutil.which("ffmpeg"):
+        logger.warning("⚠️ FFmpeg not found. Skipping stitch.")
+        return None
 
+    logger.info(f"🧵 Stitching: {path_a} + {path_b} + {path_c}")
     try:
-        # 1. Normalize all inputs to identical format
         norm_a = normalize_video(path_a)
         norm_b = normalize_video(path_b)
         norm_c = normalize_video(path_c)
         
-        # 2. Create list file for concat
+        if not all([norm_a, norm_b, norm_c]):
+            raise Exception("Normalization failed")
+        
         list_file = "concat_list.txt"
         with open(list_file, "w") as f:
             f.write(f"file '{norm_a}'\n")
             f.write(f"file '{norm_b}'\n")
             f.write(f"file '{norm_c}'\n")
         
-        # 3. Concatenate using stream copy (fast & safe)
         cmd = [
             "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", list_file,
             "-c", "copy", output_path
         ]
         subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30)
         
-        # Cleanup temps
+        # Cleanup
         for p in [norm_a, norm_b, norm_c, list_file]:
             if os.path.exists(p): os.remove(p)
             
@@ -91,7 +98,7 @@ def stitch_videos(path_a, path_b, path_c, output_path):
         
     except Exception as e:
         logger.error(f"Stitch Logic Failed: {e}")
-        raise e
+        return None  # Return None so the pipeline continues without crashing
 
 def update_job_status(job_id, status, progress, log=None, video_url=None, merged_video_url=None):
     if not job_id: return
@@ -99,14 +106,12 @@ def update_job_status(job_id, status, progress, log=None, video_url=None, merged
 
     final_url = video_url
     final_merged_url = merged_video_url
-    # Move Bridge
     if video_url and os.path.exists(video_url) and status == "completed":
         final_filename = f"{job_id}_bridge.mp4"
         dest = os.path.join("outputs", final_filename)
         if os.path.abspath(video_url) != os.path.abspath(dest): shutil.move(video_url, dest)
         final_url = f"/outputs/{final_filename}"
         if Settings.GCP_BUCKET_NAME: upload_to_gcs(dest, final_filename)
-    # Move Merged
     if merged_video_url and os.path.exists(merged_video_url) and status == "completed":
         merged_filename = f"{job_id}_merged.mp4"
         merged_dest = os.path.join("outputs", merged_filename)