Create ingest/generic_public_foia.py

ingest/generic_public_foia.py

@@ -1,37 +1,32 @@
+import abc
+import asyncio
 import time
-import aiohttp
-from ingest.generic_public_foia import GenericFOIAAdapter
-
-
-class CIAAdapter(GenericFOIAAdapter):
-    source_name = "CIA FOIA Reading Room"
-    base_url = "https://www.cia.gov/readingroom/search/site"
-    is_stub = False
-
-    def __init__(self, live: bool = False):
-        super().__init__(rate_limit=2.0)
-        self.live = live
-
-    async def search(self, query):
-        if not self.live:
-            return []
-
-        await self._throttle()
-        start = time.time()
-
-        async with aiohttp.ClientSession() as session:
-            async with session.get(
-                self.base_url,
-                params={"search_api_fulltext": query},
-                timeout=20
-            ) as resp:
-                await resp.text()
-
-        self.last_latency = time.time() - start
-
-        # Public-safe placeholder parse
-        return [{
-            "title": f"CIA FOIA document mentioning '{query}'",
-            "url": self.base_url,
-            "snippet": "Publicly released FOIA document."
-        }]
+import urllib.robotparser as robotparser
+
+class GenericFOIAAdapter(abc.ABC):
+    source_name = "UNKNOWN"
+    base_url = ""
+    is_live = False
+
+    def __init__(self):
+        self.last_call = 0
+        self.health = "unknown"
+        self._rp = robotparser.RobotFileParser()
+        self._rp.set_url(self.base_url + "/robots.txt")
+        try:
+            self._rp.read()
+        except Exception:
+            pass
+
+    async def _rate_limit(self):
+        now = time.time()
+        if now - self.last_call < 1:
+            await asyncio.sleep(1)
+        self.last_call = time.time()
+
+    def allowed(self, path="/"):
+        return self._rp.can_fetch("*", self.base_url + path)
+
+    @abc.abstractmethod
+    async def search(self, query: str):
+        pass