Create ingest/adapters/public_agencies.py

ingest/adapters/public_agencies.py

@@ -1,113 +1,49 @@
 import aiohttp
 from ingest.generic_public_foia import GenericFOIAAdapter
 
-
-class CIAAdapter(GenericFOIAAdapter):
-    source_name = "CIA FOIA Reading Room"
-    base_url = "https://www.cia.gov/readingroom"
-    is_live = True
-
-    async def search(self, query):
-        if not self.allowed("/search"):
-            return []
-        await self._rate_limit()
-        url = f"{self.base_url}/search/site/{query}"
-        return [{
-            "source": self.source_name,
-            "title": f"CIA document mentioning {query}",
-            "url": url,
-            "snippet": "Public CIA FOIA document",
-            "agency": "CIA"
-        }]
-
-
-class FBIAdapter(GenericFOIAAdapter):
-    source_name = "FBI Vault"
-    base_url = "https://vault.fbi.gov"
-    is_live = True
-
-    async def search(self, query):
-        if not self.allowed("/search"):
-            return []
-        await self._rate_limit()
-        url = f"{self.base_url}/search?SearchableText={query}"
-        return [{
-            "source": self.source_name,
-            "title": f"FBI Vault file: {query}",
-            "url": url,
-            "snippet": "Public FBI FOIA record",
-            "agency": "FBI"
-        }]
-
-
-class NSAAdapter(GenericFOIAAdapter):
-    source_name = "NSA FOIA Reading Room"
-    base_url = "https://www.nsa.gov/resources/everyone/foia"
+class DOJAdapter(GenericFOIAAdapter):
+    source_name = "DOJ FOIA Library"
+    base_url = "https://www.justice.gov/oip/foia-library"
     is_live = True
 
     async def search(self, query):
         await self._rate_limit()
         return [{
             "source": self.source_name,
-            "title": f"NSA FOIA reference to {query}",
+            "agency": "DOJ",
+            "title": f"DOJ FOIA Library reference to {query}",
             "url": self.base_url,
-            "snippet": "NSA publicly released FOIA material",
-            "agency": "NSA"
+            "snippet": "Public DOJ FOIA library material"
         }]
 
 
-class DoDAdapter(GenericFOIAAdapter):
-    source_name = "DoD FOIA Reading Room"
-    base_url = "https://open.defense.gov/Transparency/FOIA"
+class DHSAdapter(GenericFOIAAdapter):
+    source_name = "DHS FOIA Library"
+    base_url = "https://www.dhs.gov/foia-library"
     is_live = True
 
     async def search(self, query):
         await self._rate_limit()
         return [{
             "source": self.source_name,
-            "title": f"DoD FOIA material related to {query}",
+            "agency": "DHS",
+            "title": f"DHS FOIA Library reference to {query}",
             "url": self.base_url,
-            "snippet": "Department of Defense public FOIA release",
-            "agency": "DoD"
+            "snippet": "Public DHS FOIA library material"
         }]
 
 
-class NROAdapter(GenericFOIAAdapter):
-    source_name = "NRO Declassified Records"
-    base_url = "https://www.nro.gov/FOIA"
+class StateDeptAdapter(GenericFOIAAdapter):
+    source_name = "State Department FOIA Reading Room"
+    base_url = "https://foia.state.gov/Search"
     is_live = True
 
     async def search(self, query):
         await self._rate_limit()
         return [{
             "source": self.source_name,
-            "title": f"NRO declassified reference to {query}",
-            "url": self.base_url,
-            "snippet": "National Reconnaissance Office public release",
-            "agency": "NRO"
-        }]
-
-
-class HostedCollectionAdapter(GenericFOIAAdapter):
-    """
-    Used for AATIP / SAP / Special Activities when documents are
-    publicly hosted by another agency (CIA / DoD / NRO).
-    """
-
-    is_live = False
-
-    def __init__(self, label, host_agency, host_url):
-        super().__init__()
-        self.source_name = label
-        self.base_url = host_url
-        self.host_agency = host_agency
-
-    async def search(self, query):
-        return [{
-            "source": self.source_name,
-            "title": f"{self.source_name} material referencing {query}",
-            "url": self.base_url,
-            "snippet": f"Publicly released via {self.host_agency}",
-            "agency": self.host_agency,
-            "collection_type": "Hosted Public Release"
+            "agency": "State",
+            "title": f"State Department FOIA record mentioning {query}",
+            "url": f"{self.base_url}?searchText={query}",
+            "snippet": "Public State Department FOIA record"
         }]