feat: throttle VOD resolution checks to 5m and auto-complete streams older than 3 hours with no VOD

local_worker/chat_mod_worker.py

@@ -1,7 +1,8 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 import os
 import sys
 import time
+last_resolve_time = 0
 import socket
 import select
 import threading
@@ -447,9 +448,19 @@ def resolve_missing_vods():
         for stream in streams:
             stream_id = stream.get("id")
             start_time_str = stream.get("start_time")
+            end_time_str = stream.get("end_time")
             if not stream_id or not start_time_str:
                 continue
                 
+            # Calculate stream age from end_time
+            try:
+                clean_end = end_time_str.replace('Z', '+00:00') if end_time_str else start_time_str.replace('Z', '+00:00')
+                end_dt = datetime.fromisoformat(clean_end)
+                now_utc = datetime.now(timezone.utc)
+                age_hours = (now_utc - end_dt).total_seconds() / 3600.0
+            except Exception as age_err:
+                age_hours = 0
+
             matched_vod_id = find_matching_vod(start_time_str, recent_vods)
             if matched_vod_id:
                 print(f"[VOD Resolve] Stream ID {stream_id} ({start_time_str}) matched with Twitch VOD {matched_vod_id}.")
@@ -463,7 +474,14 @@ def resolve_missing_vods():
                 else:
                     print(f"[VOD Resolve] Failed to update VOD ID: {up_res.status_code}")
             else:
-                print(f"[VOD Resolve] No matching VOD found for stream ID {stream_id} ({start_time_str}) within timeframe.")
+                if age_hours > 3.0:
+                    print(f"[VOD Resolve] Stream ID {stream_id} is older than 3 hours and has no matching VOD. Marking as completed to clear queue.")
+                    try:
+                        requests.post(f"{API_URL}/api/streams/mark-backfilled", json={"streamId": stream_id}, headers=headers, timeout=10)
+                    except Exception as mark_e:
+                        print(f"[VOD Resolve] Error marking stream as completed: {mark_e}")
+                else:
+                    print(f"[VOD Resolve] No matching VOD found for stream ID {stream_id} ({start_time_str}) within timeframe (will retry).")
     except Exception as e:
         print(f"[VOD Resolve] Exception during resolve loop: {e}")
 
@@ -490,8 +508,12 @@ def run_backfill_loop():
     """Main loop to check and backfill VOD chat comments ONLY"""
     
     while not stop_flag.is_set():
-        # Try to resolve any missing VOD IDs before checking pending backfills
-        resolve_missing_vods()
+        # Try to resolve any missing VOD IDs once every 5 minutes (300s) to avoid log spam
+        global last_resolve_time
+        now_sec = time.time()
+        if now_sec - last_resolve_time > 300:
+            resolve_missing_vods()
+            last_resolve_time = now_sec
         print(f"[Backfill] Checking for pending VOD backfill tasks...")
         processed_streams = load_processed_streams()
         

local_worker/worker.py

@@ -21,8 +21,9 @@ if added_paths:
     else:
         os.environ["LD_LIBRARY_PATH"] = ":".join(added_paths)
 
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 import time
+last_resolve_time = 0
 import socket
 import select
 import threading
@@ -413,9 +414,19 @@ def resolve_missing_vods():
         for stream in streams:
             stream_id = stream.get("id")
             start_time_str = stream.get("start_time")
+            end_time_str = stream.get("end_time")
             if not stream_id or not start_time_str:
                 continue
                 
+            # Calculate stream age from end_time
+            try:
+                clean_end = end_time_str.replace('Z', '+00:00') if end_time_str else start_time_str.replace('Z', '+00:00')
+                end_dt = datetime.fromisoformat(clean_end)
+                now_utc = datetime.now(timezone.utc)
+                age_hours = (now_utc - end_dt).total_seconds() / 3600.0
+            except Exception as age_err:
+                age_hours = 0
+
             matched_vod_id = find_matching_vod(start_time_str, recent_vods)
             if matched_vod_id:
                 print(f"[VOD Resolve] Stream ID {stream_id} ({start_time_str}) matched with Twitch VOD {matched_vod_id}.")
@@ -429,7 +440,14 @@ def resolve_missing_vods():
                 else:
                     print(f"[VOD Resolve] Failed to update VOD ID: {up_res.status_code}")
             else:
-                print(f"[VOD Resolve] No matching VOD found for stream ID {stream_id} ({start_time_str}) within timeframe.")
+                if age_hours > 3.0:
+                    print(f"[VOD Resolve] Stream ID {stream_id} is older than 3 hours and has no matching VOD. Marking as completed to clear queue.")
+                    try:
+                        requests.post(f"{API_URL}/api/streams/mark-backfilled", json={"streamId": stream_id}, headers=headers, timeout=10)
+                    except Exception as mark_e:
+                        print(f"[VOD Resolve] Error marking stream as completed: {mark_e}")
+                else:
+                    print(f"[VOD Resolve] No matching VOD found for stream ID {stream_id} ({start_time_str}) within timeframe (will retry).")
     except Exception as e:
         print(f"[VOD Resolve] Exception during resolve loop: {e}")
 
@@ -532,8 +550,12 @@ def run_stream_capture(model):
     
     while not stop_flag.is_set():
         if not check_stream_live():
-            # Try to resolve any missing VOD IDs before checking pending backfills
-            resolve_missing_vods()
+            # Try to resolve any missing VOD IDs once every 5 minutes (300s) to avoid log spam
+            global last_resolve_time
+            now_sec = time.time()
+            if now_sec - last_resolve_time > 300:
+                resolve_missing_vods()
+                last_resolve_time = now_sec
             print(f"[Capture] Channel '{TWITCH_CHANNEL}' is offline. Checking for pending VOD backfill tasks...")
             try:
                 headers_get = {"x-api-key": API_KEY}