Sync from GitHub · 2026-10-20 12:42 UTC

news/pipeline.py

@@ -19,6 +19,26 @@ from news.classify import classify_item
 from news.windows import compute_expiry
 from utils.supabase_client import get_supabase
 
+# Three-way routing by confidence (replaces the old complete/incomplete split).
+# A complete, high-confidence item is business-relevant enough to publish with
+# no human step. A low-confidence item — incomplete AND the model wasn't sure
+# about it either — is almost always noise (celebrity gossip, a scraped nav
+# menu) and gets discarded automatically rather than wasting a reviewer's time.
+# Everything in between still needs a human: either it's complete but the
+# model wasn't confident, or it's incomplete but confident enough that a
+# person might be able to fill the gap.
+AUTO_PUBLISH_CONFIDENCE = 0.8
+AUTO_DISCARD_CONFIDENCE = 0.5
+
+
+def _route(result) -> str:
+    confidence = result.confidence if result.confidence is not None else 0.0
+    if result.is_complete and confidence >= AUTO_PUBLISH_CONFIDENCE:
+        return "classified"
+    if not result.is_complete and confidence < AUTO_DISCARD_CONFIDENCE:
+        return "rejected"
+    return "needs_review"
+
 
 def start_poll(since_days: int, triggered_by: str) -> str:
     """Records the poll row and hands back its id immediately — the scrape+classify
@@ -39,7 +59,7 @@ def execute_poll(poll_id: str, since_days: int) -> dict:
     db = get_supabase()
     cutoff = datetime.now(timezone.utc) - timedelta(days=since_days)
 
-    counts = {"items_scraped": 0, "items_deduped_out": 0, "items_classified": 0, "items_flagged": 0}
+    counts = {"items_scraped": 0, "items_deduped_out": 0, "items_classified": 0, "items_flagged": 0, "items_rejected": 0}
     error: str | None = None
 
     try:
@@ -72,7 +92,7 @@ def execute_poll(poll_id: str, since_days: int) -> dict:
 
                 scraped_at = datetime.now(timezone.utc)
                 expires_at = compute_expiry(result.category, scraped_at, result.deadline)
-                status = "classified" if result.is_complete else "needs_review"
+                status = _route(result)
 
                 db.table("news_items").insert({
                     "source_id":     source["id"],
@@ -96,6 +116,8 @@ def execute_poll(poll_id: str, since_days: int) -> dict:
 
                 if status == "classified":
                     counts["items_classified"] += 1
+                elif status == "rejected":
+                    counts["items_rejected"] += 1
                 else:
                     counts["items_flagged"] += 1
 

routes/news.py

@@ -52,13 +52,62 @@ def poll_news(body: PollRequest, background_tasks: BackgroundTasks, user: dict =
     return {"poll_id": poll_id, "started": True}
 
 
+@router.get("/admin/overview")
+def admin_overview(user: dict = Depends(verify_firebase_token)):
+    _verify_admin(user)
+    db = get_supabase()
+
+    def count(status: str) -> int:
+        res = db.table("news_items").select("id", count="exact").eq("status", status).execute()
+        return res.count or 0
+
+    sources = db.table("news_sources").select("active").execute().data
+    last_poll = (
+        db.table("news_polls")
+        .select("id, started_at, finished_at, error")
+        .order("started_at", desc=True)
+        .limit(1)
+        .execute()
+        .data
+    )
+
+    return {
+        "published":    count("classified"),
+        "needs_review": count("needs_review"),
+        "rejected":     count("rejected"),
+        "sources_active": sum(1 for s in sources if s["active"]),
+        "sources_total":  len(sources),
+        "last_poll": last_poll[0] if last_poll else None,
+    }
+
+
+@router.get("/admin/sources")
+def list_sources(user: dict = Depends(verify_firebase_token)):
+    _verify_admin(user)
+    db = get_supabase()
+    sources = db.table("news_sources").select("*").order("name").execute().data
+    return {"sources": sources}
+
+
+class SourceUpdate(BaseModel):
+    active: bool
+
+
+@router.patch("/admin/sources/{source_id}")
+def update_source(source_id: str, body: SourceUpdate, user: dict = Depends(verify_firebase_token)):
+    _verify_admin(user)
+    db = get_supabase()
+    db.table("news_sources").update({"active": body.active}).eq("id", source_id).execute()
+    return {"success": True}
+
+
 @router.get("/admin/polls")
 def list_polls(user: dict = Depends(verify_firebase_token)):
     _verify_admin(user)
     db = get_supabase()
     polls = (
         db.table("news_polls")
-        .select("id, triggered_by, since_days, started_at, finished_at, items_scraped, items_deduped_out, items_classified, items_flagged, error")
+        .select("id, triggered_by, since_days, started_at, finished_at, items_scraped, items_deduped_out, items_classified, items_flagged, items_rejected, error")
         .order("started_at", desc=True)
         .limit(10)
         .execute()
@@ -84,6 +133,25 @@ def list_items(business_type: str | None = None, user: dict = Depends(verify_fir
     return {"items": query.execute().data}
 
 
+@router.get("/admin/items")
+def admin_list_items(
+    status: str | None = None,
+    limit: int = 50,
+    offset: int = 0,
+    user: dict = Depends(verify_firebase_token),
+):
+    _verify_admin(user)
+    db = get_supabase()
+    query = db.table("news_items").select(
+        "id, title, url, impact, category, business_types, location, confidence, status, scraped_at, published_at, expires_at",
+        count="exact",
+    ).order("scraped_at", desc=True).range(offset, offset + limit - 1)
+    if status:
+        query = query.eq("status", status)
+    res = query.execute()
+    return {"items": res.data, "total": res.count or 0}
+
+
 @router.get("/admin/review")
 def list_review_queue(user: dict = Depends(verify_firebase_token)):
     _verify_admin(user)

sql/news_schema.sql

@@ -85,5 +85,8 @@ create table if not exists news_polls (
   items_deduped_out  int not null default 0,
   items_classified   int not null default 0,
   items_flagged      int not null default 0,
+  items_rejected     int not null default 0,  -- auto-discarded: incomplete and the model wasn't confident either
   error              text
 );
+
+alter table news_polls add column if not exists items_rejected int not null default 0;

tests/test_news.py

@@ -14,6 +14,8 @@ import pytest
 from news.dedup import url_hash
 from news.windows import compute_expiry, CATEGORY_WINDOW_DAYS, DEFAULT_WINDOW_DAYS
 from news.business_types import valid_subset, VALID_IDS
+from news.classify import ClassificationResult
+from news.pipeline import _route
 
 
 # ─── dedup ──────────────────────────────────────────────────────────────────
@@ -130,3 +132,38 @@ def test_classification_result_malformed_deadline_is_ignored():
         "deadline": "not-a-date",
     })
     assert result.deadline is None
+
+
+# ─── confidence routing ─────────────────────────────────────────────────────
+
+_SOME_VALID_ID = next(iter(VALID_IDS))
+
+
+def _result(impact="threat", category="weather", business_types=None, location="Lagos", confidence=None):
+    business_types = [_SOME_VALID_ID] if business_types is None else business_types
+    return ClassificationResult({
+        "impact": impact, "category": category,
+        "business_types": business_types, "location": location,
+        "confidence": confidence,
+    })
+
+
+def test_route_auto_publishes_complete_high_confidence():
+    assert _route(_result(confidence=0.9)) == "classified"
+
+
+def test_route_sends_complete_low_confidence_to_review():
+    assert _route(_result(confidence=0.6)) == "needs_review"
+
+
+def test_route_auto_discards_incomplete_low_confidence():
+    # e.g. celebrity gossip the model correctly isn't confident about
+    assert _route(_result(category=None, business_types=(), location=None, confidence=0.2)) == "rejected"
+
+
+def test_route_sends_incomplete_but_confident_to_review():
+    assert _route(_result(location=None, confidence=0.7)) == "needs_review"
+
+
+def test_route_treats_missing_confidence_as_zero():
+    assert _route(_result(category=None, business_types=(), location=None, confidence=None)) == "rejected"