Update ingest/fbi_vault.py

ingest/fbi_vault.py

@@ -1,29 +1,35 @@
-import requests
-from bs4 import BeautifulSoup
+import aiohttp
+from ingest.base_adapter import BaseAdapter
 
-def ingest_fbi_vault(url: str) -> dict:
-    r = requests.get(url, timeout=10)
-    r.raise_for_status()
-    soup = BeautifulSoup(r.text, "html.parser")
+class FBIAdapter(BaseAdapter):
+    name = "FBI Vault"
+    source_type = "live"
+    base_url = "https://vault.fbi.gov"
 
-    text = soup.get_text(separator=" ", strip=True)
-    title = soup.find("h1")
+    async def search(self, query: str):
+        params = {"q": query}
+        results = []
 
-    return {
-        "source": "FBI Vault",
-        "agency": "FBI",
-        "url": url,
-        "title": title.text if title else "FBI Vault Document",
-        "text": text[:10000]
-    }(r.text, "html.parser")
+        async with aiohttp.ClientSession() as session:
+            async with session.get(
+                f"{self.base_url}/search",
+                params=params,
+                timeout=15
+            ) as resp:
+                if resp.status != 200:
+                    return []
 
-    title = soup.find("h1")
-    body = soup.get_text(separator=" ", strip=True)
+                text = await resp.text()
 
-    return {
-        "source": "FBI Vault",
-        "url": url,
-        "title": title.text if title else "Untitled FBI Vault Document",
-        "text": body,
-        "agency": "FBI"
-    }
+        # Minimal, safe parsing (no deep scrape)
+        for line in text.splitlines():
+            if "/reading-room/" in line:
+                results.append({
+                    "title": f"FBI Vault result for {query}",
+                    "url": self.base_url,
+                    "snippet": "Public FBI Vault FOIA document",
+                    "source": self.name,
+                    "live": True
+                })
+
+        return results