Sync from GitHub · 2026-10-20 00:01 UTC

.env.example

@@ -10,3 +10,9 @@ ALLOWED_ORIGINS=http://localhost:3000,https://venthrive.com
 # Optional: Pinecone (used in Stage 6 when AI Chat is built)
 PINECONE_API_KEY=
 PINECONE_INDEX_NAME=venthrive-bff
+
+# Business news system (ADR-0015) — run backend/sql/news_schema.sql against
+# this Supabase project first. Service-role key: server-side only, never send
+# it to the frontend.
+SUPABASE_URL=
+SUPABASE_SERVICE_ROLE_KEY=

main.py

@@ -14,7 +14,7 @@ from fastapi.responses import JSONResponse
 import firebase_admin
 from firebase_admin import credentials
 
-from routes import bff, chat, transcribe, documents, consultation, planning, admin, account, help as help_routes
+from routes import bff, chat, transcribe, documents, consultation, planning, admin, account, help as help_routes, news
 
 
 # ─── Firebase Admin SDK initialisation ───────────────────────────────────────
@@ -92,6 +92,7 @@ app.include_router(planning.router,     prefix="/api/planning",     tags=["Plann
 app.include_router(admin.router,        prefix="/api/admin",        tags=["Admin"])
 app.include_router(account.router,      prefix="/api/account",      tags=["Account"])
 app.include_router(help_routes.router,  prefix="/api/help",         tags=["Help"])
+app.include_router(news.router,         prefix="/api/news",         tags=["News"])
 
 
 # ─── Health check (unauthenticated) ──────────────────────────────────────────

news/business_types.py

@@ -0,0 +1,23 @@
+"""
+Subsector ids the news classifier is allowed to use for `business_types`.
+Sourced from data/sectors.ts via scripts/export-subsectors.mjs — re-run that
+script and this file picks up the change automatically, no code edit needed.
+See ADR-0015 for why subsector-level (not sector or niche) was chosen.
+"""
+
+import json
+from pathlib import Path
+
+_PATH = Path(__file__).resolve().parent / "subsectors.json"
+SUBSECTORS: list[dict] = json.loads(_PATH.read_text(encoding="utf-8"))
+VALID_IDS: set[str] = {s["id"] for s in SUBSECTORS}
+
+
+def labels_for_prompt() -> str:
+    """'id — label' lines, for embedding the allowed list in the classifier prompt."""
+    return "\n".join(f"- {s['id']} — {s['label']}" for s in SUBSECTORS)
+
+
+def valid_subset(ids: list[str]) -> list[str]:
+    """Drops any id the model hallucinated that isn't in our taxonomy."""
+    return [i for i in ids if i in VALID_IDS]

news/classify.py

@@ -0,0 +1,86 @@
+"""
+Classifies one scraped item's own text directly — no scraping of other
+platforms that already classify this kind of news (see the news-system design
+discussion referenced in ADR-0015). A constrained-enum + few-shot prompt sent
+to the light (20B) Groq model, modeled on DataWiz-Project's bounded,
+admin-validated "autofill" step.
+"""
+
+import json
+from datetime import date
+
+from utils.groq_client import call_groq_json, LIGHT_MODEL
+from news.business_types import labels_for_prompt, valid_subset
+
+IMPACTS = ("threat", "opportunity", "neutral")
+CATEGORIES = ("policy", "weather", "infrastructure", "funding", "security", "other")
+
+_SYSTEM_PROMPT = f"""You classify a single news/bulletin item for VenThrive, a Nigerian small-business platform.
+
+Read the title and text given, then output ONLY a JSON object with these exact fields:
+- "impact": one of {list(IMPACTS)}
+- "category": one of {list(CATEGORIES)}
+- "business_types": a list of 1-4 ids from the taxonomy below that this item affects. Empty list if none clearly apply.
+- "location": the Nigerian state, region, or "Nationwide" this affects. Null if genuinely unclear.
+- "deadline": an ISO date (YYYY-MM-DD) ONLY if impact is "opportunity" AND the text states an explicit application/closing deadline. Otherwise null.
+- "confidence": your confidence in this classification, 0.0 to 1.0.
+
+Business type taxonomy (use these ids exactly, do not invent new ones):
+{labels_for_prompt()}
+
+Examples:
+- A new CBN forex directive raising import costs -> impact=threat, category=policy, business_types=[relevant import/trade subsectors], location=Nationwide
+- A flood warning for Lagos -> impact=threat, category=weather, business_types=[subsectors with physical premises/stock in the area], location=Lagos
+- A state SME grant with an application deadline of 2026-12-01 -> impact=opportunity, category=funding, deadline=2026-12-01
+- A tax holiday for new small businesses with no stated end date -> impact=opportunity, category=policy, deadline=null
+
+If the item is not genuinely business-relevant, or you cannot determine a category, business
+type, or location with reasonable confidence, leave the relevant field null/empty rather than
+guessing — an incomplete classification is flagged for human review instead of shown to users,
+which is the correct outcome for a low-confidence item."""
+
+
+class ClassificationResult:
+    def __init__(self, raw: dict):
+        impact = raw.get("impact")
+        category = raw.get("category")
+        business_types = valid_subset(raw.get("business_types") or [])
+        location = (raw.get("location") or "").strip() or None
+        confidence = raw.get("confidence")
+
+        deadline_str = raw.get("deadline")
+        deadline: date | None = None
+        if deadline_str:
+            try:
+                deadline = date.fromisoformat(deadline_str)
+            except ValueError:
+                deadline = None  # malformed date from the model — treat as not given
+
+        self.impact = impact if impact in IMPACTS else None
+        self.category = category if category in CATEGORIES else None
+        self.business_types = business_types
+        self.location = location
+        self.deadline = deadline if self.impact == "opportunity" else None
+        self.confidence = float(confidence) if isinstance(confidence, (int, float)) else None
+
+    @property
+    def is_complete(self) -> bool:
+        """The ADR-0015 completeness bar: category + >=1 business type + location."""
+        return bool(self.category and self.business_types and self.location)
+
+    def review_reason(self) -> str | None:
+        if self.is_complete:
+            return None
+        missing = []
+        if not self.category: missing.append("category")
+        if not self.business_types: missing.append("business_types")
+        if not self.location: missing.append("location")
+        return f"Missing: {', '.join(missing)}"
+
+
+def classify_item(title: str, text: str, fixture: str | None = None) -> ClassificationResult:
+    user_prompt = f"Title: {title}\n\nText: {text[:4000]}"
+    raw = call_groq_json(_SYSTEM_PROMPT, user_prompt, model=LIGHT_MODEL, max_tokens=500, fixture=fixture)
+    if isinstance(raw, str):
+        raw = json.loads(raw)
+    return ClassificationResult(raw)

news/dedup.py

@@ -0,0 +1,12 @@
+"""URL-hash dedup (ADR-0015: dedup runs before classification, so we never
+spend a Groq call on something we're about to discard as already-seen)."""
+
+import hashlib
+
+
+def url_hash(url: str) -> str:
+    # Trailing slashes and query strings are common no-op variants of the same
+    # article URL — strip them so the same piece of content doesn't get
+    # rescraped as "new" on a second poll just because of a tracking param.
+    normalized = url.strip().rstrip("/").split("?", 1)[0].split("#", 1)[0]
+    return hashlib.sha256(normalized.encode("utf-8")).hexdigest()

news/pipeline.py

@@ -0,0 +1,104 @@
+"""
+Orchestrates one admin-triggered poll (ADR-0015): scrape every active source,
+dedup by URL hash, classify everything that survives dedup, apply the
+completeness bar, and record a run summary in news_polls.
+
+A single source failing to fetch (network error, bad feed, site down) must not
+fail the whole poll — it's logged and skipped, same as DataWiz's approach to
+per-source scrape failures. A single item failing to classify (Groq error,
+malformed response) is likewise skipped rather than aborting the run; it stays
+unclassified and will be retried on the next poll since it's never marked
+'rejected'.
+"""
+
+from datetime import datetime, timedelta, timezone
+
+from news.scrape import fetch_source
+from news.dedup import url_hash
+from news.classify import classify_item
+from news.windows import compute_expiry
+from utils.supabase_client import get_supabase
+
+
+def run_poll(since_days: int, triggered_by: str) -> dict:
+    db = get_supabase()
+    now = datetime.now(timezone.utc)
+    cutoff = now - timedelta(days=since_days)
+
+    poll = db.table("news_polls").insert({
+        "triggered_by": triggered_by,
+        "since_days": since_days,
+        "started_at": now.isoformat(),
+    }).execute()
+    poll_id = poll.data[0]["id"]
+
+    counts = {"items_scraped": 0, "items_deduped_out": 0, "items_classified": 0, "items_flagged": 0}
+    error: str | None = None
+
+    try:
+        sources = db.table("news_sources").select("*").eq("active", True).execute().data
+
+        for source in sources:
+            try:
+                raw_items = fetch_source(source)
+            except Exception as e:
+                print(f"[news] source '{source['name']}' failed to fetch: {e}")
+                continue
+
+            for item in raw_items:
+                if item.published_at and item.published_at < cutoff:
+                    continue
+
+                h = url_hash(item.url)
+                existing = db.table("news_items").select("id").eq("url_hash", h).limit(1).execute().data
+                if existing:
+                    counts["items_deduped_out"] += 1
+                    continue
+
+                counts["items_scraped"] += 1
+
+                try:
+                    result = classify_item(item.title, item.text)
+                except Exception as e:
+                    print(f"[news] classification failed for {item.url}: {e}")
+                    continue
+
+                scraped_at = datetime.now(timezone.utc)
+                expires_at = compute_expiry(result.category, scraped_at, result.deadline)
+                status = "classified" if result.is_complete else "needs_review"
+
+                db.table("news_items").insert({
+                    "source_id":     source["id"],
+                    "url":           item.url,
+                    "url_hash":      h,
+                    "title":         item.title,
+                    "raw_text":      item.text,
+                    "published_at":  item.published_at.isoformat() if item.published_at else None,
+                    "scraped_at":    scraped_at.isoformat(),
+                    "status":        status,
+                    "impact":        result.impact,
+                    "category":      result.category,
+                    "business_types": result.business_types,
+                    "location":      result.location,
+                    "deadline":      result.deadline.isoformat() if result.deadline else None,
+                    "expires_at":    expires_at.isoformat(),
+                    "confidence":    result.confidence,
+                    "review_reason": result.review_reason(),
+                    "classified_at": scraped_at.isoformat(),
+                }).execute()
+
+                if status == "classified":
+                    counts["items_classified"] += 1
+                else:
+                    counts["items_flagged"] += 1
+
+    except Exception as e:
+        error = str(e)
+
+    db.table("news_polls").update({
+        "finished_at": datetime.now(timezone.utc).isoformat(),
+        "error": error,
+        **counts,
+    }).eq("id", poll_id).execute()
+
+    return {"poll_id": poll_id, "error": error, **counts}

news/scrape.py

@@ -0,0 +1,83 @@
+"""
+Fetches raw items from a single source. Two strategies (ADR-0015):
+  - 'rss':  feedparser, for the national/economic news sources that actually
+            publish a feed.
+  - 'html': a generic title + first-substantial-paragraph extractor, for
+            government bulletins and funder/program pages that don't publish
+            RSS. Expect a higher needs_review rate from these by design — the
+            completeness bar in classify.py catches weak extractions rather
+            than showing them to users.
+
+Network errors for one source must never take down a whole poll run — the
+caller (pipeline.py) treats a per-source failure as "zero items from this
+source this run", not a fatal error.
+"""
+
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from time import mktime
+
+import feedparser
+import httpx
+from bs4 import BeautifulSoup
+
+REQUEST_TIMEOUT = 15.0
+USER_AGENT = "VenThriveNewsBot/1.0 (+https://venthrive.com; business-news aggregator)"
+
+
+@dataclass
+class RawItem:
+    url: str
+    title: str
+    text: str
+    published_at: datetime | None
+
+
+def fetch_rss(feed_url: str) -> list[RawItem]:
+    parsed = feedparser.parse(
+        feed_url,
+        agent=USER_AGENT,
+        request_headers={"User-Agent": USER_AGENT},
+    )
+    items: list[RawItem] = []
+    for entry in parsed.entries:
+        url = entry.get("link", "").strip()
+        title = entry.get("title", "").strip()
+        if not url or not title:
+            continue
+        text = entry.get("summary", "") or entry.get("description", "")
+        text = BeautifulSoup(text, "html.parser").get_text(" ", strip=True)
+        published_at = None
+        if entry.get("published_parsed"):
+            published_at = datetime.fromtimestamp(mktime(entry.published_parsed), tz=timezone.utc)
+        items.append(RawItem(url=url, title=title, text=text, published_at=published_at))
+    return items
+
+
+def fetch_html(page_url: str) -> list[RawItem]:
+    """One item: the page itself. Government/funder pages are usually a single
+    announcement or a landing page we read as-is — there's no reliable, generic
+    way to discover "articles" on an arbitrary site's markup, so this reads the
+    page's own headline + body rather than trying to crawl links from it."""
+    resp = httpx.get(page_url, headers={"User-Agent": USER_AGENT}, timeout=REQUEST_TIMEOUT, follow_redirects=True)
+    resp.raise_for_status()
+    soup = BeautifulSoup(resp.text, "html.parser")
+
+    title_tag = soup.find("h1") or soup.find("title")
+    title = title_tag.get_text(strip=True) if title_tag else ""
+
+    paragraphs = [p.get_text(" ", strip=True) for p in soup.find_all("p")]
+    substantial = [p for p in paragraphs if len(p) > 60]
+    text = " ".join(substantial[:5])
+
+    if not title or not text:
+        return []
+    return [RawItem(url=page_url, title=title, text=text, published_at=None)]
+
+
+def fetch_source(source: dict) -> list[RawItem]:
+    if source["source_type"] == "rss":
+        return fetch_rss(source["url"])
+    if source["source_type"] == "html":
+        return fetch_html(source["url"])
+    raise ValueError(f"Unknown source_type: {source['source_type']!r}")

news/subsectors.json

@@ -0,0 +1,198 @@
+[
+  {
+    "id": "catering_food_service",
+    "label": "Catering & Food Service"
+  },
+  {
+    "id": "food_production_processing",
+    "label": "Food Production & Processing"
+  },
+  {
+    "id": "agricultural_production",
+    "label": "Agricultural Production"
+  },
+  {
+    "id": "agri_trade_distribution",
+    "label": "Agricultural Trade & Distribution"
+  },
+  {
+    "id": "clothing_apparel",
+    "label": "Clothing & Apparel"
+  },
+  {
+    "id": "tailoring_bespoke",
+    "label": "Tailoring & Bespoke"
+  },
+  {
+    "id": "footwear",
+    "label": "Footwear"
+  },
+  {
+    "id": "accessories_jewellery",
+    "label": "Accessories & Jewellery"
+  },
+  {
+    "id": "software_apps",
+    "label": "Software & Apps"
+  },
+  {
+    "id": "digital_services",
+    "label": "Digital Services"
+  },
+  {
+    "id": "hardware_repairs",
+    "label": "Hardware & Repairs"
+  },
+  {
+    "id": "it_consulting",
+    "label": "IT Consulting & Support"
+  },
+  {
+    "id": "healthcare_services",
+    "label": "Healthcare Services"
+  },
+  {
+    "id": "wellness_fitness",
+    "label": "Wellness & Fitness"
+  },
+  {
+    "id": "hair_services",
+    "label": "Hair Services"
+  },
+  {
+    "id": "skincare_cosmetics",
+    "label": "Skincare & Cosmetics"
+  },
+  {
+    "id": "body_spa",
+    "label": "Body Care & Spa"
+  },
+  {
+    "id": "formal_education",
+    "label": "Formal Education"
+  },
+  {
+    "id": "skills_vocational",
+    "label": "Skills & Vocational Training"
+  },
+  {
+    "id": "online_coaching",
+    "label": "Online Learning & Coaching"
+  },
+  {
+    "id": "financial_services",
+    "label": "Financial Services"
+  },
+  {
+    "id": "insurance",
+    "label": "Insurance"
+  },
+  {
+    "id": "fintech_payments",
+    "label": "Fintech & Payments"
+  },
+  {
+    "id": "delivery_services",
+    "label": "Delivery Services"
+  },
+  {
+    "id": "haulage_freight",
+    "label": "Haulage & Freight"
+  },
+  {
+    "id": "passenger_transport",
+    "label": "Passenger Transport"
+  },
+  {
+    "id": "real_estate",
+    "label": "Real Estate"
+  },
+  {
+    "id": "construction_renovation",
+    "label": "Construction & Renovation"
+  },
+  {
+    "id": "building_materials",
+    "label": "Building Materials"
+  },
+  {
+    "id": "visual_arts",
+    "label": "Visual Arts"
+  },
+  {
+    "id": "content_media",
+    "label": "Content & Media"
+  },
+  {
+    "id": "performing_arts",
+    "label": "Performing Arts"
+  },
+  {
+    "id": "event_planning",
+    "label": "Event Planning"
+  },
+  {
+    "id": "event_support",
+    "label": "Event Support Services"
+  },
+  {
+    "id": "entertainment_venues",
+    "label": "Entertainment Venues"
+  },
+  {
+    "id": "light_manufacturing",
+    "label": "Light Manufacturing"
+  },
+  {
+    "id": "industrial_manufacturing",
+    "label": "Industrial Manufacturing"
+  },
+  {
+    "id": "textiles_materials",
+    "label": "Textiles & Materials"
+  },
+  {
+    "id": "retail_trade",
+    "label": "Retail Trade"
+  },
+  {
+    "id": "wholesale_distribution",
+    "label": "Wholesale & Distribution"
+  },
+  {
+    "id": "import_export",
+    "label": "Import & Export"
+  },
+  {
+    "id": "legal_compliance",
+    "label": "Legal & Compliance"
+  },
+  {
+    "id": "accounting_finance_advisory",
+    "label": "Accounting & Finance Advisory"
+  },
+  {
+    "id": "management_consulting",
+    "label": "Management & Business Consulting"
+  },
+  {
+    "id": "vehicle_maintenance",
+    "label": "Vehicle Maintenance & Repair"
+  },
+  {
+    "id": "vehicle_trade",
+    "label": "Vehicle Trade & Rentals"
+  },
+  {
+    "id": "cleaning_services",
+    "label": "Cleaning Services"
+  },
+  {
+    "id": "home_services",
+    "label": "Home & Household Services"
+  },
+  {
+    "id": "other_services",
+    "label": "Other Services"
+  }
+]

news/windows.py

@@ -0,0 +1,30 @@
+"""
+Default relevance windows per category (ADR-0015). These are starting points,
+not final — day counts picked before any real classified history existed, to
+be revisited once there's enough data to see how fast each category actually
+goes stale.
+
+An opportunity with an explicit deadline in the source text uses that deadline
+instead of its category's default window — see classify.py.
+"""
+
+from datetime import datetime, timedelta, timezone, date
+
+CATEGORY_WINDOW_DAYS = {
+    "weather":         7,
+    "infrastructure":  14,
+    "security":        14,
+    "policy":          90,
+    "funding":         30,
+    "other":           21,
+}
+
+DEFAULT_WINDOW_DAYS = 21
+
+
+def compute_expiry(category: str | None, scraped_at: datetime, deadline: date | None) -> datetime:
+    """The point at which an item stops being shown to users."""
+    if deadline is not None:
+        return datetime.combine(deadline, datetime.min.time(), tzinfo=timezone.utc)
+    days = CATEGORY_WINDOW_DAYS.get(category or "", DEFAULT_WINDOW_DAYS)
+    return scraped_at + timedelta(days=days)

requirements.txt

@@ -10,3 +10,8 @@ httpx==0.27.2
 pinecone>=9.0.0
 # PDF text extraction for book ingestion script (pure Python, Windows-compatible)
 pypdf>=4.0.0
+# Business news system (ADR-0015): Supabase storage, RSS/HTML scraping
+supabase>=2.9.0,<3.0.0
+feedparser>=6.0.11,<7.0.0
+beautifulsoup4>=4.12.3,<5.0.0
+

routes/news.py

@@ -0,0 +1,121 @@
+"""
+Business News Routes (ADR-0015)
+──────────────────────────────────────────────────────────────────────────────
+Routes:
+  POST /api/news/poll              → admin-triggered scrape+classify run (max 30 days back)
+  GET  /api/news/items             → classified, unexpired items (optionally filtered by business type)
+  GET  /api/news/admin/review      → admin: items flagged incomplete, awaiting a human fix
+  PATCH /api/news/admin/review/{id} → admin: manually complete or reject a flagged item
+"""
+
+from datetime import datetime, timezone
+
+from fastapi import APIRouter, HTTPException, Depends
+from pydantic import BaseModel, Field
+from firebase_admin import firestore
+from auth import verify_firebase_token
+from news.pipeline import run_poll
+from news.classify import IMPACTS, CATEGORIES
+from news.business_types import valid_subset
+from utils.supabase_client import get_supabase
+
+router = APIRouter()
+
+
+def _verify_admin(user: dict) -> str:
+    uid = user.get("uid")
+    if not uid:
+        raise HTTPException(status_code=401, detail="Unauthorized.")
+    db   = firestore.client()
+    doc  = db.collection("user_data").document(uid).get()
+    role = doc.to_dict().get("role", "user") if doc.exists else "user"
+    if role != "admin":
+        raise HTTPException(status_code=403, detail="Admin access required.")
+    return uid
+
+
+class PollRequest(BaseModel):
+    since_days: int = Field(default=7, ge=1, le=30)
+
+
+@router.post("/poll")
+def poll_news(body: PollRequest, user: dict = Depends(verify_firebase_token)):
+    admin_uid = _verify_admin(user)
+    try:
+        result = run_poll(since_days=body.since_days, triggered_by=admin_uid)
+    except Exception as e:
+        raise HTTPException(status_code=502, detail=f"Poll failed: {e}")
+    return result
+
+
+@router.get("/items")
+def list_items(business_type: str | None = None, user: dict = Depends(verify_firebase_token)):
+    db = get_supabase()
+    now_iso = datetime.now(timezone.utc).isoformat()
+    query = (
+        db.table("news_items")
+        .select("id, title, url, impact, category, business_types, location, deadline, expires_at, published_at")
+        .eq("status", "classified")
+        .gt("expires_at", now_iso)
+        .order("published_at", desc=True)
+        .limit(100)
+    )
+    if business_type:
+        query = query.contains("business_types", [business_type])
+    return {"items": query.execute().data}
+
+
+@router.get("/admin/review")
+def list_review_queue(user: dict = Depends(verify_firebase_token)):
+    _verify_admin(user)
+    db = get_supabase()
+    items = (
+        db.table("news_items")
+        .select("id, title, url, raw_text, impact, category, business_types, location, review_reason, scraped_at")
+        .eq("status", "needs_review")
+        .order("scraped_at", desc=True)
+        .limit(100)
+        .execute()
+        .data
+    )
+    return {"items": items}
+
+
+class ReviewFix(BaseModel):
+    status:         str = Field(pattern="^(classified|rejected)$")
+    impact:         str | None = None
+    category:       str | None = None
+    business_types: list[str] | None = None
+    location:       str | None = None
+
+
+@router.patch("/admin/review/{item_id}")
+def fix_review_item(item_id: str, body: ReviewFix, user: dict = Depends(verify_firebase_token)):
+    _verify_admin(user)
+    db = get_supabase()
+
+    if body.status == "rejected":
+        db.table("news_items").update({"status": "rejected"}).eq("id", item_id).execute()
+        return {"success": True, "status": "rejected"}
+
+    # Completing by hand still has to satisfy the same bar automated classification does.
+    if body.impact and body.impact not in IMPACTS:
+        raise HTTPException(status_code=400, detail=f"impact must be one of {IMPACTS}")
+    if body.category and body.category not in CATEGORIES:
+        raise HTTPException(status_code=400, detail=f"category must be one of {CATEGORIES}")
+
+    business_types = valid_subset(body.business_types or [])
+    if not (body.category and body.location and business_types):
+        raise HTTPException(status_code=400, detail="category, location and at least one valid business_type are required to mark this classified.")
+
+    db.table("news_items").update({
+        "status":         "classified",
+        "impact":         body.impact,
+        "category":       body.category,
+        "business_types": business_types,
+        "location":       body.location,
+        "review_reason":  None,
+        "classified_at":  datetime.now(timezone.utc).isoformat(),
+    }).eq("id", item_id).execute()
+
+    return {"success": True, "status": "classified"}

sql/news_schema.sql

@@ -0,0 +1,89 @@
+-- VenThrive business-news system — run this once against your Supabase project's
+-- SQL editor (or `psql`). See docs/blueprint/adr/0015-business-news-system-design.md
+-- for the reasoning behind every choice below.
+
+create extension if not exists pgcrypto;
+
+-- ─── Sources ────────────────────────────────────────────────────────────────
+-- Sources are data, not code, so an admin can add/disable one without a deploy.
+-- 'rss' sources are polled with feedparser; 'html' sources with a generic
+-- title + first-paragraph extractor (see backend/news/scrape.py).
+create table if not exists news_sources (
+  id            uuid primary key default gen_random_uuid(),
+  name          text not null,
+  url           text not null unique,
+  source_type   text not null check (source_type in ('rss', 'html')),
+  -- Which of the three sourcing strategies this feeds (ADR-0015) — purely
+  -- informational, doesn't change scrape behavior.
+  source_group  text not null check (source_group in ('national_news', 'agency_bulletin', 'funder_program')),
+  active        boolean not null default true,
+  created_at    timestamptz not null default now()
+);
+
+-- A handful of real, well-known starter sources per ADR-0015's three source
+-- groups. Add more through this table directly — no code change needed.
+insert into news_sources (name, url, source_type, source_group) values
+  ('BusinessDay',               'https://businessday.ng/feed/',                 'rss',  'national_news'),
+  ('Nairametrics',              'https://nairametrics.com/feed/',               'rss',  'national_news'),
+  ('Premium Times — Business',  'https://www.premiumtimesng.com/category/business/feed', 'rss', 'national_news'),
+  ('NiMet — Weather Alerts',    'https://www.nimet.gov.ng',                     'html', 'agency_bulletin'),
+  ('NEMA — Advisories',         'https://nema.gov.ng',                          'html', 'agency_bulletin'),
+  ('SMEDAN',                    'https://smedan.gov.ng',                        'html', 'funder_program'),
+  ('Bank of Industry — News',   'https://www.boi.ng/category/news/',            'html', 'funder_program'),
+  ('Tony Elumelu Foundation',   'https://www.tonyelumelufoundation.org/news',   'html', 'funder_program')
+on conflict (url) do nothing;
+
+-- ─── Items ──────────────────────────────────────────────────────────────────
+create table if not exists news_items (
+  id              uuid primary key default gen_random_uuid(),
+  source_id       uuid references news_sources(id) on delete set null,
+  url             text not null,
+  url_hash        text not null unique,
+  title           text not null,
+  raw_text        text not null default '',
+  published_at    timestamptz,
+  scraped_at      timestamptz not null default now(),
+
+  status          text not null default 'pending'
+                    check (status in ('pending', 'classified', 'needs_review', 'rejected')),
+
+  -- Classification output (ADR-0015) — all null until classified.
+  impact          text check (impact in ('threat', 'opportunity', 'neutral')),
+  category        text check (category in ('policy', 'weather', 'infrastructure', 'funding', 'security', 'other')),
+  business_types  text[],          -- subsector ids from data/sectors.ts, see backend/news/subsectors.json
+  location        text,
+  deadline        date,            -- only set for an opportunity with an explicit deadline in the source text
+  expires_at      timestamptz,     -- deadline if set, else scraped_at + the category's default window
+  confidence      numeric,
+  review_reason   text,            -- why this needs_review, for the admin queue
+  classified_at   timestamptz,
+
+  -- The completeness bar from ADR-0015, enforced here, not just in application
+  -- code: a classified item must have a category, at least one business type,
+  -- and a location.
+  constraint classified_items_are_complete check (
+    status <> 'classified'
+    or (category is not null and location is not null
+        and business_types is not null and array_length(business_types, 1) > 0)
+  )
+);
+
+create index if not exists news_items_status_idx on news_items (status);
+create index if not exists news_items_expires_at_idx on news_items (expires_at);
+create index if not exists news_items_business_types_idx on news_items using gin (business_types);
+
+-- ─── Poll run log ───────────────────────────────────────────────────────────
+-- One row per admin-triggered poll (ADR-0015: admin-triggered only, daily
+-- auto-poll deferred) — an audit trail and a place to surface run-level errors.
+create table if not exists news_polls (
+  id                 uuid primary key default gen_random_uuid(),
+  triggered_by       text not null,   -- admin's Firebase UID
+  since_days         int not null check (since_days between 1 and 30),
+  started_at         timestamptz not null default now(),
+  finished_at        timestamptz,
+  items_scraped      int not null default 0,
+  items_deduped_out  int not null default 0,
+  items_classified   int not null default 0,
+  items_flagged      int not null default 0,
+  error              text
+);

tests/fixtures/groq/news_classify_complete.json

@@ -0,0 +1,8 @@
+{
+  "impact": "threat",
+  "category": "weather",
+  "business_types": ["catering_food_service", "food_production_processing"],
+  "location": "Lagos",
+  "deadline": null,
+  "confidence": 0.9
+}

tests/fixtures/groq/news_classify_incomplete.json

@@ -0,0 +1,8 @@
+{
+  "impact": "neutral",
+  "category": null,
+  "business_types": [],
+  "location": null,
+  "deadline": null,
+  "confidence": 0.3
+}

tests/fixtures/groq/news_classify_opportunity.json

@@ -0,0 +1,8 @@
+{
+  "impact": "opportunity",
+  "category": "funding",
+  "business_types": ["catering_food_service"],
+  "location": "Nationwide",
+  "deadline": "2026-12-01",
+  "confidence": 0.85
+}

tests/test_groq_fixture_mode.py

@@ -52,6 +52,41 @@ def test_call_groq_transcribe_returns_object_with_text(fixture_mode):
     assert "meals a day" in result.text
 
 
+@pytest.fixture
+def demo_mode(monkeypatch):
+    monkeypatch.setenv("DEMO_MODE", "1")
+    monkeypatch.setenv("E2E_FIXTURE_MODE", "1")
+    import utils.groq_client as gc
+    importlib.reload(gc)
+    yield gc
+    monkeypatch.delenv("E2E_FIXTURE_MODE", raising=False)
+    monkeypatch.delenv("DEMO_MODE", raising=False)
+    importlib.reload(gc)
+
+
+def test_demo_mode_turns_json_text_into_test_environment_but_keeps_structure(demo_mode):
+    data = demo_mode.call_groq_json("system", "user", fixture="planning_tasks")
+    tasks = data["suggestions"]
+    assert tasks and all(t["title"] == "Test Environment" for t in tasks)
+    assert all(t["column"] in {"todo", "in_progress", "done"} for t in tasks)
+
+
+def test_demo_mode_stream_says_test_environment(demo_mode):
+    chunks = list(demo_mode.call_groq_stream("system", "user", fixture="chat_response"))
+    assert "".join(c.choices[0].delta.content for c in chunks).strip() == "Test Environment"
+
+
+def test_demo_mode_needs_fixture_mode():
+    os.environ["DEMO_MODE"] = "1"
+    try:
+        import utils.groq_client as gc
+        importlib.reload(gc)
+        assert gc.DEMO_MODE is False
+    finally:
+        os.environ.pop("DEMO_MODE", None)
+        importlib.reload(gc)
+
+
 def test_fixture_mode_is_off_by_default():
     import utils.groq_client as gc
     importlib.reload(gc)

tests/test_news.py

@@ -0,0 +1,132 @@
+"""
+Unit tests for the business news system (ADR-0015): dedup hashing, relevance
+windows, business-type validation, and classification completeness logic.
+
+classify_item goes through real Groq in production; here it's exercised only
+in E2E_FIXTURE_MODE against canned fixtures, same pattern as
+test_groq_fixture_mode.py — no network calls.
+"""
+import importlib
+from datetime import date, datetime, timezone
+
+import pytest
+
+from news.dedup import url_hash
+from news.windows import compute_expiry, CATEGORY_WINDOW_DAYS, DEFAULT_WINDOW_DAYS
+from news.business_types import valid_subset, VALID_IDS
+
+
+# ─── dedup ──────────────────────────────────────────────────────────────────
+
+def test_url_hash_ignores_trailing_slash():
+    assert url_hash("https://example.com/a") == url_hash("https://example.com/a/")
+
+
+def test_url_hash_ignores_query_string():
+    assert url_hash("https://example.com/a") == url_hash("https://example.com/a?utm_source=x")
+
+
+def test_url_hash_ignores_fragment():
+    assert url_hash("https://example.com/a") == url_hash("https://example.com/a#section")
+
+
+def test_url_hash_distinguishes_different_urls():
+    assert url_hash("https://example.com/a") != url_hash("https://example.com/b")
+
+
+# ─── windows ────────────────────────────────────────────────────────────────
+
+def test_compute_expiry_uses_category_window():
+    scraped_at = datetime(2026, 1, 1, tzinfo=timezone.utc)
+    expiry = compute_expiry("weather", scraped_at, None)
+    assert expiry == scraped_at + __import__("datetime").timedelta(days=CATEGORY_WINDOW_DAYS["weather"])
+
+
+def test_compute_expiry_falls_back_to_default_for_unknown_category():
+    scraped_at = datetime(2026, 1, 1, tzinfo=timezone.utc)
+    expiry = compute_expiry(None, scraped_at, None)
+    assert expiry == scraped_at + __import__("datetime").timedelta(days=DEFAULT_WINDOW_DAYS)
+
+
+def test_compute_expiry_deadline_overrides_category_window():
+    scraped_at = datetime(2026, 1, 1, tzinfo=timezone.utc)
+    deadline = date(2026, 12, 1)
+    expiry = compute_expiry("funding", scraped_at, deadline)
+    assert expiry == datetime(2026, 12, 1, tzinfo=timezone.utc)
+
+
+# ─── business_types ───────────────────────────────────────────────────────
+
+def test_valid_subset_drops_hallucinated_ids():
+    real_id = next(iter(VALID_IDS))
+    assert valid_subset([real_id, "not-a-real-subsector"]) == [real_id]
+
+
+def test_valid_subset_empty_input():
+    assert valid_subset([]) == []
+
+
+# ─── classify (fixture mode) ───────────────────────────────────────────────
+
+@pytest.fixture
+def classify_module(monkeypatch):
+    monkeypatch.setenv("E2E_FIXTURE_MODE", "1")
+    import utils.groq_client as gc
+    importlib.reload(gc)
+    import news.classify as classify
+    importlib.reload(classify)
+    yield classify
+    monkeypatch.delenv("E2E_FIXTURE_MODE", raising=False)
+    importlib.reload(gc)
+    importlib.reload(classify)
+
+
+def test_classify_item_complete_result_is_complete(classify_module):
+    result = classify_module.classify_item("Lagos flood warning", "text", fixture="news_classify_complete")
+    assert result.is_complete
+    assert result.review_reason() is None
+    assert result.impact == "threat"
+    assert result.category == "weather"
+
+
+def test_classify_item_incomplete_result_flags_missing_fields(classify_module):
+    result = classify_module.classify_item("Unclear item", "text", fixture="news_classify_incomplete")
+    assert not result.is_complete
+    reason = result.review_reason()
+    assert "category" in reason
+    assert "business_types" in reason
+    assert "location" in reason
+
+
+def test_classify_item_opportunity_keeps_deadline(classify_module):
+    result = classify_module.classify_item("State SME grant", "text", fixture="news_classify_opportunity")
+    assert result.impact == "opportunity"
+    assert result.deadline == date(2026, 12, 1)
+
+
+def test_classify_item_deadline_dropped_when_not_opportunity(classify_module):
+    result = classify_module.classify_item("Lagos flood warning", "text", fixture="news_classify_complete")
+    assert result.deadline is None
+
+
+def test_classification_result_drops_invalid_enum_values():
+    result = __import__("news.classify", fromlist=["ClassificationResult"]).ClassificationResult({
+        "impact": "not-a-real-impact",
+        "category": "not-a-real-category",
+        "business_types": [],
+        "location": "Lagos",
+    })
+    assert result.impact is None
+    assert result.category is None
+
+
+def test_classification_result_malformed_deadline_is_ignored():
+    from news.classify import ClassificationResult
+    result = ClassificationResult({
+        "impact": "opportunity",
+        "category": "funding",
+        "business_types": [],
+        "location": "Lagos",
+        "deadline": "not-a-date",
+    })
+    assert result.deadline is None

utils/groq_client.py

@@ -36,6 +36,27 @@ FIXTURE_MODE = os.environ.get("E2E_FIXTURE_MODE") == "1"
 _FIXTURES_DIR = Path(__file__).resolve().parent.parent / "tests" / "fixtures" / "groq"
 
 
+# DEMO_MODE=1 (with fixture mode, used by `npm run demo`): every AI answer reads
+# "Test Environment". Only fields the app needs to keep working are left alone.
+DEMO_MODE = FIXTURE_MODE and os.environ.get("DEMO_MODE") == "1"
+DEMO_TEXT = "Test Environment"
+_DEMO_KEEP_KEYS = {"id", "type", "column", "targetDate", "date", "status"}
+
+
+def _demo_value(value, key=None):
+    if isinstance(value, dict):
+        return {k: _demo_value(v, k) for k, v in value.items()}
+    if isinstance(value, list):
+        strings = [v for v in value if isinstance(v, str)]
+        if strings and len(strings) == len(value):
+            # Numbered so choice lists and React keys stay distinct
+            return [f"{DEMO_TEXT} {i + 1}" for i in range(len(value))] if len(value) > 1 else [DEMO_TEXT]
+        return [_demo_value(v, key) for v in value]
+    if isinstance(value, str) and key not in _DEMO_KEEP_KEYS:
+        return DEMO_TEXT
+    return value
+
+
 def _load_json_fixture(name: str) -> dict:
     path = _FIXTURES_DIR / f"{name}.json"
     if not path.exists():
@@ -43,7 +64,8 @@ def _load_json_fixture(name: str) -> dict:
             f"[E2E_FIXTURE_MODE] no fixture file for '{name}' at {path} — "
             "add one under backend/tests/fixtures/groq/"
         )
-    return json.loads(path.read_text(encoding="utf-8"))
+    data = json.loads(path.read_text(encoding="utf-8"))
+    return _demo_value(data) if DEMO_MODE else data
 
 
 def _load_text_fixture(name: str) -> str:
@@ -53,7 +75,7 @@ def _load_text_fixture(name: str) -> str:
             f"[E2E_FIXTURE_MODE] no fixture file for '{name}' at {path} — "
             "add one under backend/tests/fixtures/groq/"
         )
-    return path.read_text(encoding="utf-8")
+    return DEMO_TEXT if DEMO_MODE else path.read_text(encoding="utf-8")
 
 # ─── Load all keys ────────────────────────────────────────────────────────────
 # Accepts: GROQ_API_KEY, GROQ_API_KEY_1 ... GROQ_API_KEY_N (any suffix)

utils/supabase_client.py

@@ -0,0 +1,32 @@
+"""
+Supabase client for the business-news system (ADR-0015). Lazily initialized —
+unlike groq_client.py's eager key check, this only raises if something
+actually tries to use Supabase without it being configured, so importing this
+module (or any module that imports it) never breaks the test suite or routes
+that don't touch news.
+
+Uses the service-role key: this client is server-side only (backend/news/*,
+backend/routes/news.py) and must never be sent to or used from the frontend.
+"""
+
+import os
+from supabase import create_client, Client
+
+_client: Client | None = None
+
+
+def get_supabase() -> Client:
+    global _client
+    if _client is not None:
+        return _client
+
+    url = os.environ.get("SUPABASE_URL")
+    key = os.environ.get("SUPABASE_SERVICE_ROLE_KEY")
+    if not url or not key:
+        raise RuntimeError(
+            "SUPABASE_URL / SUPABASE_SERVICE_ROLE_KEY not set — the news system needs a "
+            "Supabase project (see backend/sql/news_schema.sql and ADR-0015) before "
+            "/api/news/* routes can do anything."
+        )
+    _client = create_client(url, key)
+    return _client