Update ingest/generic_public_foia.py

ingest/generic_public_foia.py

@@ -1,46 +1,38 @@
 # ingest/generic_public_foia.py
 
+import abc
+import asyncio
 import requests
-from bs4 import BeautifulSoup
-from urllib.parse import urlparse
-from typing import Dict
-from .agency_registry import ALLOWED_FOIA_SOURCES
+from typing import List, Dict
 
-MAX_CHARS = 12000
 
-def infer_agency_from_url(url: str) -> str:
-    host = urlparse(url).netloc.lower()
-
-    for domain, agency in ALLOWED_FOIA_SOURCES.items():
-        if domain.startswith("label:"):
-            continue
-        if domain in host:
-            return agency
-
-    return "Unknown"
-
-def ingest_public_foia_url(url: str) -> Dict:
+class GenericFOIAAdapter(abc.ABC):
     """
-    SAFE ingestion:
-    - user-supplied URL only
-    - public FOIA / reading room pages
-    - no crawling, no discovery
+    Base adapter for all public FOIA Electronic Reading Rooms.
+    Enforces rate-limiting, public-only access, and safe defaults.
     """
 
-    agency = infer_agency_from_url(url)
-
-    r = requests.get(url, timeout=15)
-    r.raise_for_status()
-
-    soup = BeautifulSoup(r.text, "html.parser")
-
-    title = soup.find("h1")
-    text = soup.get_text(separator=" ", strip=True)
-
-    return {
-        "agency": agency,
-        "url": url,
-        "title": title.text.strip() if title else "Public FOIA Document",
-        "text": text[:MAX_CHARS],
-        "source_type": "public_foia"
-    }
+    source_name: str = "UNKNOWN"
+    base_url: str = ""
+
+    def __init__(self, rate_limit_seconds: float = 1.0):
+        self.rate_limit_seconds = rate_limit_seconds
+        self._last_call = 0.0
+
+    async def _rate_limit(self):
+        delta = asyncio.get_event_loop().time() - self._last_call
+        if delta < self.rate_limit_seconds:
+            await asyncio.sleep(self.rate_limit_seconds - delta)
+        self._last_call = asyncio.get_event_loop().time()
+
+    @abc.abstractmethod
+    async def search(self, query: str) -> List[Dict]:
+        """
+        Perform a public FOIA search.
+        Must return a list of dicts with:
+        - source
+        - title
+        - url
+        - snippet
+        """
+        raise NotImplementedError