Upload 31 files

ingest/aatip_reading_room.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class AATIPAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "AATIP"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/agency_registry.py

@@ -0,0 +1,39 @@
+# ingest/agency_registry.py
+
+# Domains are PUBLIC FOIA / reading room hosts only.
+# Labels may include units if (and only if) a public FOIA page exists
+# and the user provides its URL.
+
+ALLOWED_FOIA_SOURCES = {
+    # Core
+    "vault.fbi.gov": "FBI",
+    "www.cia.gov": "CIA",
+    "www.archives.gov": "NARA",
+    "foia.state.gov": "State Dept",
+    "www.nsa.gov": "NSA",
+    "www.defense.gov": "DoD",
+    "www.esd.whs.mil": "DoD FOIA",
+    "www.whitehouse.gov": "White House",
+
+    # Military (public FOIA pages)
+    "www.af.mil": "USAF",
+    "www.navy.mil": "US Navy",
+    "www.marines.mil": "USMC",
+    "www.army.mil": "US Army",
+    "www.spaceforce.mil": "US Space Force",
+
+    # Intelligence / defense components (public FOIA pages only)
+    "www.dia.mil": "DIA",
+    "www.nro.gov": "NRO",
+
+    # Law enforcement / protective services (public FOIA pages)
+    "www.secretservice.gov": "US Secret Service",
+    "www.dea.gov": "DEA",
+
+    # Labels for historical / organizational references
+    # (NO claim of dedicated public repositories)
+    # These are ONLY labels applied if a public FOIA URL is supplied.
+    "label:SAC": "CIA Special Activities Center (label only)",
+    "label:SAD": "CIA Special Activities Division (label only)",
+    "label:NIA": "National Intelligence Authority (historical)"
+}

ingest/air_force_foia_reading_room.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class USAirForceAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "USAirForce"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/army_foia_reading_room.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class USArmyAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "USArmy"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/cia_reading_room.py

@@ -0,0 +1,21 @@
+import httpx
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class CIAAdapter(GenericFOIAAdapter):
+    name = "CIA"
+    rate_limit = 1
+    robots_respected = True
+    base_url = "https://www.cia.gov/readingroom/search/site/"
+
+    async def search(self, query: str):
+        async with httpx.AsyncClient(timeout=10) as client:
+            r = await client.get(self.base_url, params={"query": query})
+            if r.status_code != 200:
+                return []
+            # Minimal safe parse: return page-level hit
+            return [{
+                "source": "CIA FOIA Reading Room",
+                "query": query,
+                "url": str(r.url),
+                "snippet": "Public FOIA search result page"
+            }]

ingest/coast_guard_foia_reading_room.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class USCoastGuardAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "USCoastGuard"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/darpa_reading_room.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class DARPAAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "DARPA"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/dhs_reading_room.py

@@ -0,0 +1,6 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class DHSAdapter(GenericFOIAAdapter):
+    name = "DHS"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/dia_reading_room.py

@@ -0,0 +1,6 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class DIAAdapter(GenericFOIAAdapter):
+    name = "DIA"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/dod_reading_room.py

@@ -0,0 +1,6 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class DoDAdapter(GenericFOIAAdapter):
+    name = "DoD"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/dod_reading_room_live.py

@@ -0,0 +1,20 @@
+import httpx
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class DoDAdapter(GenericFOIAAdapter):
+    name = "DoD FOIA Reading Room"
+    rate_limit = 1
+    robots_respected = True
+    base_url = "https://www.esd.whs.mil/FOIA/Reading-Room/"
+
+    async def search(self, query: str):
+        async with httpx.AsyncClient(timeout=10) as client:
+            r = await client.get(self.base_url, params={"search": query})
+            if r.status_code != 200:
+                return []
+            return [{
+                "source": "DoD FOIA Reading Room",
+                "query": query,
+                "url": str(r.url),
+                "snippet": "Public DoD FOIA reading room page"
+            }]

ingest/fbi_vault.py

@@ -0,0 +1,29 @@
+import requests
+from bs4 import BeautifulSoup
+
+def ingest_fbi_vault(url: str) -> dict:
+    r = requests.get(url, timeout=10)
+    r.raise_for_status()
+    soup = BeautifulSoup(r.text, "html.parser")
+
+    text = soup.get_text(separator=" ", strip=True)
+    title = soup.find("h1")
+
+    return {
+        "source": "FBI Vault",
+        "agency": "FBI",
+        "url": url,
+        "title": title.text if title else "FBI Vault Document",
+        "text": text[:10000]
+    }(r.text, "html.parser")
+
+    title = soup.find("h1")
+    body = soup.get_text(separator=" ", strip=True)
+
+    return {
+        "source": "FBI Vault",
+        "url": url,
+        "title": title.text if title else "Untitled FBI Vault Document",
+        "text": body,
+        "agency": "FBI"
+    }

ingest/fbi_vault_live.py

@@ -0,0 +1,20 @@
+import httpx
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class FBIAdapter(GenericFOIAAdapter):
+    name = "FBI Vault"
+    rate_limit = 1
+    robots_respected = True
+    base_url = "https://vault.fbi.gov/search"
+
+    async def search(self, query: str):
+        async with httpx.AsyncClient(timeout=10) as client:
+            r = await client.get(self.base_url, params={"q": query})
+            if r.status_code != 200:
+                return []
+            return [{
+                "source": "FBI Vault",
+                "query": query,
+                "url": str(r.url),
+                "snippet": "Public FBI Vault search results"
+            }]

ingest/generic_public_foia.py

@@ -0,0 +1,46 @@
+# ingest/generic_public_foia.py
+
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urlparse
+from typing import Dict
+from .agency_registry import ALLOWED_FOIA_SOURCES
+
+MAX_CHARS = 12000
+
+def infer_agency_from_url(url: str) -> str:
+    host = urlparse(url).netloc.lower()
+
+    for domain, agency in ALLOWED_FOIA_SOURCES.items():
+        if domain.startswith("label:"):
+            continue
+        if domain in host:
+            return agency
+
+    return "Unknown"
+
+def ingest_public_foia_url(url: str) -> Dict:
+    """
+    SAFE ingestion:
+    - user-supplied URL only
+    - public FOIA / reading room pages
+    - no crawling, no discovery
+    """
+
+    agency = infer_agency_from_url(url)
+
+    r = requests.get(url, timeout=15)
+    r.raise_for_status()
+
+    soup = BeautifulSoup(r.text, "html.parser")
+
+    title = soup.find("h1")
+    text = soup.get_text(separator=" ", strip=True)
+
+    return {
+        "agency": agency,
+        "url": url,
+        "title": title.text.strip() if title else "Public FOIA Document",
+        "text": text[:MAX_CHARS],
+        "source_type": "public_foia"
+    }

ingest/ice_reading_room.py

@@ -0,0 +1,6 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class ICEAdapter(GenericFOIAAdapter):
+    name = "ICE"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/loader.py

@@ -0,0 +1,24 @@
+import requests
+from bs4 import BeautifulSoup
+from typing import List, Dict
+
+def ingest_documents(enable_scraping: bool = False) -> List[Dict]:
+    if not enable_scraping:
+        return []
+
+    # HF-safe: capped, read-only metadata fetch
+    docs = []
+    try:
+        r = requests.get("https://vault.fbi.gov", timeout=10)
+        soup = BeautifulSoup(r.text, "html.parser")
+        for link in soup.select("a")[:10]:
+            docs.append({
+                "title": link.text.strip(),
+                "agency": "FBI",
+                "date": "",
+                "content": link.get("href", "")
+            })
+    except Exception:
+        pass
+
+    return docs

ingest/marine_corps_foia_reading_room.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class USMarineCorpsAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "USMarineCorps"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/navy_foia_reading_room.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class USNavyAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "USNavy"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/nia_reading_room.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class NIAAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "NIA"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/nis_reading_room.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class NISAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "NIS"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/nro_reading_room.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class NROAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "NRO"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/nsa_reading_room.py

@@ -0,0 +1,6 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class NSAAdapter(GenericFOIAAdapter):
+    name = "NSA"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/sap_public_releases.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class SpecialAccessProgramsAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "SpecialAccessPrograms"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/secret_service_reading_room.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class SecretServiceAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "SecretService"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/sources.py

@@ -0,0 +1,9 @@
+AGENCY_SOURCES = {
+    "FBI": "https://vault.fbi.gov",
+    "CIA": "https://www.cia.gov/readingroom",
+    "DoD": "https://www.esd.whs.mil/FOIA/",
+    "NSA": "https://www.nsa.gov/Helpful-Links/FOIA/",
+    "NRO": "https://www.nro.gov/FOIA/",
+    "USAF": "https://www.afhra.af.mil/FOIA/",
+    "White House": "https://www.archives.gov/foia"
+}

ingest/space_force_foia_reading_room.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class USSpaceForceAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "USSpaceForce"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/special_activities_public.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class SpecialActivitiesAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "SpecialActivities"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/special_projects_public.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class SpecialProjectsAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "SpecialProjects"
+    rate_limit = 1  # requests per second
+    robots_respected = True

ingest/tencap_reading_room.py

@@ -0,0 +1,9 @@
+from ingest.generic_public_foia import GenericFOIAAdapter
+
+class TENCAPAdapter(GenericFOIAAdapter):
+    """Public-release FOIA reading room adapter.
+    NOTE: This adapter is restricted to publicly released materials only.
+    """
+    name = "TENCAP"
+    rate_limit = 1  # requests per second
+    robots_respected = True