Sync from GitHub · 2026-10-20 08:32 UTC

news/pipeline.py

@@ -20,17 +20,24 @@ from news.windows import compute_expiry
 from utils.supabase_client import get_supabase
 
 
-def run_poll(since_days: int, triggered_by: str) -> dict:
+def start_poll(since_days: int, triggered_by: str) -> str:
+    """Records the poll row and hands back its id immediately — the scrape+classify
+    work for even a handful of sources takes minutes (Groq calls per item), which
+    is well past most gateway timeouts (HF Spaces included). The caller runs
+    execute_poll(poll_id, since_days) as a background task rather than awaiting
+    it inline, so the HTTP response never has to wait on it."""
     db = get_supabase()
-    now = datetime.now(timezone.utc)
-    cutoff = now - timedelta(days=since_days)
-
     poll = db.table("news_polls").insert({
         "triggered_by": triggered_by,
         "since_days": since_days,
-        "started_at": now.isoformat(),
+        "started_at": datetime.now(timezone.utc).isoformat(),
     }).execute()
-    poll_id = poll.data[0]["id"]
+    return poll.data[0]["id"]
+
+
+def execute_poll(poll_id: str, since_days: int) -> dict:
+    db = get_supabase()
+    cutoff = datetime.now(timezone.utc) - timedelta(days=since_days)
 
     counts = {"items_scraped": 0, "items_deduped_out": 0, "items_classified": 0, "items_flagged": 0}
     error: str | None = None

routes/news.py

@@ -3,18 +3,24 @@ Business News Routes (ADR-0015)
 ──────────────────────────────────────────────────────────────────────────────
 Routes:
   POST /api/news/poll              → admin-triggered scrape+classify run (max 30 days back)
+  GET  /api/news/admin/polls       → admin: recent poll runs and their outcome
   GET  /api/news/items             → classified, unexpired items (optionally filtered by business type)
   GET  /api/news/admin/review      → admin: items flagged incomplete, awaiting a human fix
   PATCH /api/news/admin/review/{id} → admin: manually complete or reject a flagged item
+
+Polling runs as a background task (see news/pipeline.py) because scraping every
+source and classifying each new item through Groq takes minutes — far past any
+gateway's request timeout. POST /poll returns as soon as the run is recorded;
+the frontend watches GET /admin/polls for it to finish.
 """
 
 from datetime import datetime, timezone
 
-from fastapi import APIRouter, HTTPException, Depends
+from fastapi import APIRouter, HTTPException, Depends, BackgroundTasks
 from pydantic import BaseModel, Field
 from firebase_admin import firestore
 from auth import verify_firebase_token
-from news.pipeline import run_poll
+from news.pipeline import start_poll, execute_poll
 from news.classify import IMPACTS, CATEGORIES
 from news.business_types import valid_subset
 from utils.supabase_client import get_supabase
@@ -39,13 +45,26 @@ class PollRequest(BaseModel):
 
 
 @router.post("/poll")
-def poll_news(body: PollRequest, user: dict = Depends(verify_firebase_token)):
+def poll_news(body: PollRequest, background_tasks: BackgroundTasks, user: dict = Depends(verify_firebase_token)):
     admin_uid = _verify_admin(user)
-    try:
-        result = run_poll(since_days=body.since_days, triggered_by=admin_uid)
-    except Exception as e:
-        raise HTTPException(status_code=502, detail=f"Poll failed: {e}")
-    return result
+    poll_id = start_poll(since_days=body.since_days, triggered_by=admin_uid)
+    background_tasks.add_task(execute_poll, poll_id, body.since_days)
+    return {"poll_id": poll_id, "started": True}
+
+
+@router.get("/admin/polls")
+def list_polls(user: dict = Depends(verify_firebase_token)):
+    _verify_admin(user)
+    db = get_supabase()
+    polls = (
+        db.table("news_polls")
+        .select("id, triggered_by, since_days, started_at, finished_at, items_scraped, items_deduped_out, items_classified, items_flagged, error")
+        .order("started_at", desc=True)
+        .limit(10)
+        .execute()
+        .data
+    )
+    return {"polls": polls}
 
 
 @router.get("/items")