fix: replace in-memory store with SQLite so scraper and UI share data across workers

app.py

@@ -1,8 +1,10 @@
 # -*- coding: utf-8 -*-
 """
 app.py — Hugging Face Spaces adaptation of frontend/streamlit_app.py
-All features identical; infrastructure layer uses in-memory deque store
+All features identical; infrastructure layer uses SQLite store
 and threading instead of Redis + subprocess.
+SQLite is used so the scraper thread and Streamlit UI share the same data
+even when running in a multi-worker environment.
 """
 
 import streamlit as st
@@ -15,37 +17,52 @@ import re
 import os
 import threading
 import logging
+import sqlite3
 from collections import deque, defaultdict
 from datetime import datetime, timedelta
 
-# ── In-memory store (replaces Redis) ─────────────────────────────────────────
+# ── SQLite store (replaces in-memory deque) ───────────────────────────────────
+# Stored in /tmp so it persists for the lifetime of the container process
+DB_PATH = "/tmp/livepulse.db"
 MAX_STORE_MESSAGES = 10000
 
-_STORE_LOCK: threading.Lock = threading.Lock()
-_STORE: dict[str, deque] = {}          # keyed by redis_key string
-_META:  dict[str, str]   = {}          # misc key-value (e.g. "video_title")
+_DB_LOCK = threading.Lock()
+_META: dict[str, str] = {}   # misc key-value (e.g. "video_title", "scraper_error")
 
 # Scraper thread registry
 _SCRAPER_THREADS: dict[str, threading.Thread] = {}
 _SCRAPER_STOP:    dict[str, threading.Event]  = {}
 
 
-def _get_deque(key: str) -> deque:
-    """Return (creating if needed) the deque for a given key."""
-    if key not in _STORE:
-        _STORE[key] = deque(maxlen=MAX_STORE_MESSAGES)
-    return _STORE[key]
+def _get_db() -> sqlite3.Connection:
+    """Return a thread-local SQLite connection."""
+    conn = sqlite3.connect(DB_PATH, check_same_thread=False)
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS messages (
+            id      INTEGER PRIMARY KEY AUTOINCREMENT,
+            key     TEXT NOT NULL,
+            value   TEXT NOT NULL
+        )
+    """)
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_key ON messages(key)")
+    conn.commit()
+    return conn
+
+
+# Initialize DB on import
+_db_conn = _get_db()
 
 
-# Redis-compatible helpers
 def store_lrange(key: str, start: int, end: int) -> list[str]:
-    """Emulate r.lrange(key, start, end)."""
-    with _STORE_LOCK:
-        d = list(_get_deque(key))
-    n = len(d)
+    """Emulate r.lrange(key, start, end) — returns rows in insertion order."""
+    with _DB_LOCK:
+        rows = _db_conn.execute(
+            "SELECT value FROM messages WHERE key=? ORDER BY id ASC", (key,)
+        ).fetchall()
+    values = [r[0] for r in rows]
+    n = len(values)
     if n == 0:
         return []
-    # Normalise negative indices
     if start < 0:
         start = max(n + start, 0)
     if end < 0:
@@ -53,23 +70,35 @@ def store_lrange(key: str, start: int, end: int) -> list[str]:
     end = min(end, n - 1)
     if start > end:
         return []
-    return d[start : end + 1]
+    return values[start: end + 1]
 
 
 def store_llen(key: str) -> int:
-    with _STORE_LOCK:
-        return len(_get_deque(key))
+    with _DB_LOCK:
+        row = _db_conn.execute(
+            "SELECT COUNT(*) FROM messages WHERE key=?", (key,)
+        ).fetchone()
+    return row[0] if row else 0
 
 
 def store_delete(key: str) -> None:
-    with _STORE_LOCK:
-        if key in _STORE:
-            _STORE[key].clear()
+    with _DB_LOCK:
+        _db_conn.execute("DELETE FROM messages WHERE key=?", (key,))
+        _db_conn.commit()
 
 
 def store_rpush(key: str, value: str) -> None:
-    with _STORE_LOCK:
-        _get_deque(key).append(value)
+    with _DB_LOCK:
+        _db_conn.execute(
+            "INSERT INTO messages (key, value) VALUES (?, ?)", (key, value)
+        )
+        # Trim to MAX_STORE_MESSAGES per key
+        _db_conn.execute("""
+            DELETE FROM messages WHERE key=? AND id NOT IN (
+                SELECT id FROM messages WHERE key=? ORDER BY id DESC LIMIT ?
+            )
+        """, (key, key, MAX_STORE_MESSAGES))
+        _db_conn.commit()
 
 
 # ── Inline config (replaces backend/config.py) ────────────────────────────────