ingest/adapters/doj.py

import aiohttp
from bs4 import BeautifulSoup
from ingest.generic_public_foia import GenericFOIAAdapter

class DOJAdapter(GenericFOIAAdapter):
    source_name = "DOJ"
    base_url = "https://www.justice.gov/oip/foia-library"

    async def search(self, query: str):
        if not self.robots_allowed():
            return []

        await self._rate_limit()

        async with aiohttp.ClientSession() as session:
            async with session.get(self.base_url) as resp:
                html = await resp.text()

        if query.lower() not in html.lower():
            return []

        return [{
            "source": self.source_name,
            "title": "DOJ FOIA Library Reference",
            "url": self.base_url,
            "snippet": f"Query '{query}' appears in DOJ FOIA Library page"
        }]