ingest/adapters/fbi.py

import aiohttp
from bs4 import BeautifulSoup
from ingest.generic_public_foia import GenericFOIAAdapter

class FBIAdapter(GenericFOIAAdapter):
    source_name = "FBI Vault"
    base_url = "https://vault.fbi.gov/search"

    async def search(self, query: str):
        if not self.robots_allowed():
            return []

        await self._rate_limit()

        params = {"SearchableText": query}

        async with aiohttp.ClientSession() as session:
            async with session.get(self.base_url, params=params) as resp:
                html = await resp.text()

        soup = BeautifulSoup(html, "html.parser")
        results = []

        for item in soup.select(".document"):
            title = item.select_one("h3")
            link = item.select_one("a")

            if not title or not link:
                continue

            results.append({
                "source": self.source_name,
                "title": title.text.strip(),
                "url": link["href"],
                "snippet": title.text.strip()
            })

        return results