fix(d1): decouple image capture from D1 push (PR #45) — unfreeze live feed

README.md

@@ -18,23 +18,25 @@ rebuilds) and caches responses at the edge (CDN + stale-while-revalidate).
 
 ## What runs here
 
-One process (`python -m sec_listener.worker`) = SEC listener + Markdown backfill
-+ FastAPI API on port **7860**, which Hugging Face exposes at the Space URL.
+One process (`python -m sec_listener.worker`) = SEC listener + Markdown / filing-header /
+image backfill + FastAPI API on port **7860**, which Hugging Face exposes at the Space URL.
 
-## Required Space secret
+## Space secrets
 
 | Secret | Purpose |
 |--------|---------|
 | `SEC_API_KEY` | Key clients must send as the `X-API-Key` header. Set it, and use the same value in the frontend's `SEC_API_KEY`. |
+| `HF_TOKEN` | Enables the parallel HF-dataset mirror, cold-start boot-restore, and scanned-exhibit image capture. |
 
 ## Endpoints
 
-`/health` (open), `/api/ex10?page=&page_size=`, `/api/ex10/since?seconds=`,
-`/api/ex10/{id}`, `/api/stats`, `/api/search?q=`.
+`/health` (open), `/api/ex10?page=&page_size=&form=&cik=&filer=&sort=`,
+`/api/ex10/since?seconds=`, `/api/ex10/{id}`, `/api/facets`, `/api/stats`, `/api/search?q=`.
 
 ## Notes
 
-- Ships with a seed database for instant content; new filings stream in
-  continuously. Enable persistent storage (mount `/data`, set
-  `SEC_DB_PATH=/data/ex10_listener.db`) to retain data across restarts.
+- Ships with a seed database for instant content; new filings stream in continuously.
+- HF Spaces have **no persistent disk**, so durability comes from the HF dataset mirror:
+  on boot the worker restores SQLite from `arthrod/sec-ex10-exhibits`
+  (`python -m sec_listener.boot_restore`) and re-mirrors every `SEC_HF_SYNC_INTERVAL` seconds.
 - Public SEC filing data only. Not legal/investment advice. Not affiliated with the SEC.

sec_listener/db.py

@@ -200,9 +200,13 @@ class Database:
             conn.commit()
 
     def update_image_urls(self, exhibit_id: int, urls: list[str]) -> None:
+        # Reset mirrored so a row that was already pushed to D1 on markdown+metadata
+        # (image capture is decoupled from the push) is re-queued for a follow-up push,
+        # propagating the freshly captured image_urls via the ingest upsert. A row not
+        # yet mirrored is unaffected (mirrored stays 0).
         with self.connect() as conn:
             conn.execute(
-                "UPDATE ex10_exhibits SET image_urls = ? WHERE id = ?",
+                "UPDATE ex10_exhibits SET image_urls = ?, mirrored = 0 WHERE id = ?",
                 (json.dumps(urls), exhibit_id),
             )
             conn.commit()

sec_listener/worker.py

@@ -199,12 +199,16 @@ class BackfillWorker:
 
 
 async def _d1_push_loop(db: Database, *, url: str, key: str, interval: float,
-                        require_images: bool = True, batch: int = 100):
+                        require_images: bool = False, batch: int = 100):
     """Periodically push *finalized* SQLite rows to D1 via the ingest route.
 
     SQLite stays the working buffer; D1 is authoritative. Failures here never
     disturb the listener/backfill — rows stay unmirrored and retry next cycle.
-    When image capture is off (no HF_TOKEN) rows finalize without image_urls.
+
+    Image capture is **decoupled** from the push: a row finalizes (and pushes) on
+    markdown + metadata alone, so a slow/stalled image backfill never freezes the
+    live feed. When images are captured later, ``update_image_urls`` re-queues the
+    row so a follow-up push propagates them to D1 via the ingest upsert.
     """
     from .d1_sync import push_finalized
 
@@ -236,11 +240,14 @@ async def _run_all(config: Config):
 
     # Push finalized rows to D1 (authoritative store) via the ingest route —
     # gated on the shared SEC_API_KEY. Without it, SQLite is the sole store.
+    # Image capture is decoupled from the push (require_images=False default): rows
+    # reach D1 on markdown+metadata, and re-push once images arrive (see
+    # update_image_urls / _d1_push_loop). This keeps the live feed flowing even when
+    # the image backfill is slow or stalled.
     if config.api_key:
         tasks.append(asyncio.create_task(
             _d1_push_loop(db, url=config.d1_ingest_url, key=config.api_key,
-                          interval=float(os.environ.get("SEC_D1_PUSH_INTERVAL", "60")),
-                          require_images=bool(os.environ.get("HF_TOKEN"))),
+                          interval=float(os.environ.get("SEC_D1_PUSH_INTERVAL", "60"))),
             name="d1-push",
         ))