ingest/adapters/dhs.py

import aiohttp
from bs4 import BeautifulSoup
from ingest.generic_public_foia import GenericFOIAAdapter

class DHSAdapter(GenericFOIAAdapter):
    source_name = "DHS"
    base_url = "https://www.dhs.gov/foia-library"

    async def search(self, query: str):
        if not self.robots_allowed():
            return []

        await self._rate_limit()

        async with aiohttp.ClientSession() as session:
            async with session.get(self.base_url) as resp:
                html = await resp.text()

        if query.lower() not in html.lower():
            return []

        return [{
            "source": self.source_name,
            "title": "DHS FOIA Library Reference",
            "url": self.base_url,
            "snippet": f"Query '{query}' appears in DHS FOIA Library page"
        }]