API quota conservation: scraper polling rate now matches UI refresh interval

app.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
 """
 app.py � Hugging Face Spaces adaptation of frontend/streamlit_app.py
 Infrastructure: SQLite store + threading scraper (no Redis, no subprocess).
@@ -209,7 +209,7 @@ def _fetch_chat_messages(live_chat_id: str, api_key: str, page_token: str | None
         return [], None, 5000
 
 
-def _scraper_thread_fn(video_id: str, redis_key: str, stop_event: threading.Event) -> None:
+def _scraper_thread_fn(video_id: str, redis_key: str, stop_event: threading.Event, min_poll_s: float = 10.0) -> None:
     """Background thread � scrapes live chat via YouTube Data API v3."""
     api_key = os.getenv("YOUTUBE_API_KEY", "")
     logger.info("YOUTUBE_API_KEY present: %s (length=%d)", bool(api_key), len(api_key))
@@ -323,14 +323,14 @@ def _scraper_thread_fn(video_id: str, redis_key: str, stop_event: threading.Even
         if len(seen_ids) > 5000:
             seen_ids = set(list(seen_ids)[-2000:])
 
-        # respect YouTube's requested polling interval (min 3s to be safe)
-        wait_s = max(poll_ms / 1000, 3.0)
+        # Respect YouTube's requested polling interval, but never faster than min_poll_s
+        wait_s = max(poll_ms / 1000, min_poll_s)
         stop_event.wait(timeout=wait_s)
 
     logger.info("Scraper thread ended � key=%s", redis_key)
 
 
-def start_scraper(slot_idx: int, video_id: str, redis_key: str) -> None:
+def start_scraper(slot_idx: int, video_id: str, redis_key: str, min_poll_s: float = 10.0) -> None:
     """Start a scraper thread for the given slot, stopping any existing one first."""
     key = str(slot_idx)
     stop_scraper(slot_idx)
@@ -338,7 +338,7 @@ def start_scraper(slot_idx: int, video_id: str, redis_key: str) -> None:
     stop_event = threading.Event()
     t = threading.Thread(
         target=_scraper_thread_fn,
-        args=(video_id, redis_key, stop_event),
+        args=(video_id, redis_key, stop_event, min_poll_s),
         daemon=True,
         name=f"scraper-{slot_idx}",
     )
@@ -997,7 +997,7 @@ with st.sidebar:
                 vid  = extract_video_id(st.session_state[vid_skey])
                 rkey = st.session_state[rkey_skey].strip() or f"chat_messages_{label.lower()}"
                 if vid:
-                    start_scraper(idx, vid, rkey)
+                    start_scraper(idx, vid, rkey, min_poll_s=float(st.session_state.get("refresh_rate", 10)))
                     st.session_state.streams[idx]["proc"]      = _SCRAPER_THREADS.get(str(idx))
                     st.session_state.streams[idx]["video_id"]  = vid
                     st.session_state.streams[idx]["redis_key"] = rkey

shared.py

@@ -208,7 +208,7 @@ def _fetch_chat_messages(live_chat_id: str, api_key: str, page_token: str | None
         return [], None, 5000
 
 
-def _scraper_thread_fn(video_id: str, redis_key: str, stop_event: threading.Event) -> None:
+def _scraper_thread_fn(video_id: str, redis_key: str, stop_event: threading.Event, min_poll_s: float = 10.0) -> None:
     api_key = os.getenv("YOUTUBE_API_KEY", "")
     if not api_key:
         msg = "YOUTUBE_API_KEY env var not set. Cannot start scraper."
@@ -291,17 +291,18 @@ def _scraper_thread_fn(video_id: str, redis_key: str, stop_event: threading.Even
         if len(seen_ids) > 5000:
             seen_ids = set(list(seen_ids)[-2000:])
 
-        wait_s = max(poll_ms / 1000, 3.0)
+        # Respect YouTube's requested polling interval, but never faster than min_poll_s
+        wait_s = max(poll_ms / 1000, min_poll_s)
         stop_event.wait(timeout=wait_s)
 
 
-def start_scraper(slot_idx: int, video_id: str, redis_key: str) -> None:
+def start_scraper(slot_idx: int, video_id: str, redis_key: str, min_poll_s: float = 10.0) -> None:
     key = str(slot_idx)
     stop_scraper(slot_idx)
     stop_event = threading.Event()
     t = threading.Thread(
         target=_scraper_thread_fn,
-        args=(video_id, redis_key, stop_event),
+        args=(video_id, redis_key, stop_event, min_poll_s),
         daemon=True,
         name=f"scraper-{slot_idx}",
     )