Upload 29 files

AGENCY_COVERAGE.md

@@ -0,0 +1,11 @@
+
+# Agency Coverage Map
+
+| Agency | Public FOIA Reading Room |
+|------|---------------------------|
+| CIA | https://www.cia.gov/readingroom/ |
+| FBI | https://vault.fbi.gov/ |
+| DoD | https://www.foia.mil/ |
+| NSA | https://www.nsa.gov/readingroom/ |
+| NRO | https://www.nro.gov/FOIA/ |
+| DHS | https://www.dhs.gov/foia-reading-room |

CODE_OF_CONDUCT.md

@@ -0,0 +1,28 @@
+# Code of Conduct
+
+## Our Pledge
+
+This project is committed to providing a respectful, inclusive, and responsible environment for all contributors and users.
+
+## Acceptable Use
+
+Participants agree to:
+- Use this project for lawful, ethical, and non-harmful purposes
+- Respect the public-record nature of FOIA documents
+- Avoid speculative, defamatory, or misleading interpretations
+
+## Unacceptable Use
+
+This project must not be used to:
+- Harass or target individuals
+- Make unsubstantiated allegations
+- Claim access to classified or restricted information
+- Bypass legal or ethical safeguards
+
+## Enforcement
+
+Maintainers may remove content or restrict access that violates this Code of Conduct.
+
+---
+
+This project is intended for civic transparency, education, and research.

Dockerfile.hf

@@ -0,0 +1,5 @@
+FROM python:3.10-slim
+WORKDIR /app
+COPY . /app
+RUN pip install --no-cache-dir -r requirements.txt
+CMD ["python", "app.py"]

ETHICS.md

@@ -0,0 +1,24 @@
+# Ethics Policy
+
+## Purpose
+
+This project exists to support transparency, research, and public understanding of government records released under the Freedom of Information Act (FOIA).
+
+## Guiding Principles
+
+- **Public Sources Only:** All data must originate from publicly released documents.
+- **No Speculation:** The project does not infer, predict, or hypothesize beyond document text.
+- **Citation First:** Outputs must be traceable to source material.
+- **No Harm:** The tool must not be used to defame, harass, or mislead.
+
+## Redactions
+
+Redacted content is respected. This project does not attempt to reconstruct or infer withheld information.
+
+## Accountability
+
+Users are responsible for how they interpret and use results. This tool provides analytical assistance, not conclusions.
+
+---
+
+Ethical transparency is foundational to this project.

FILE_INVENTORY.txt

@@ -0,0 +1,60 @@
+CODE_OF_CONDUCT.md
+ETHICS.md
+README.md
+README_PROD.md
+__init__.py
+adapters/__init__.py
+adapters/cia.py
+adapters/common.py
+adapters/dea.py
+adapters/dhs.py
+adapters/dia.py
+adapters/dod.py
+adapters/doj.py
+adapters/fbi.py
+adapters/ice.py
+adapters/nia.py
+adapters/nsa.py
+app.py
+appeal_pdf.py
+appeals/__init__.py
+appeals/pdf_appeal.py
+audit.py
+collaboration.py
+collaboration/__init__.py
+collaboration/icij.py
+core/__init__.py
+core/analysis.py
+core/appeals.py
+core/explain.py
+core/index.py
+core/multi_program.py
+core/redaction.py
+core/search.py
+core/vector.py
+data/demo/documents/aatip_sample.txt
+data/demo/documents/tencap_sample.txt
+data/demo/metadata.json
+data/foia_sources.json
+entity_graph.py
+export_utils.py
+file_structure.txt
+foia_pdf.py
+foia_requests.py
+gitattributes.txt
+ingest/__init__.py
+ingest/agency_registry.py
+ingest/cia_reading_room.py
+ingest/fbi_vault.py
+ingest/generic_public_foia.py
+ingest/loader.py
+ingest/sources.py
+requirements.txt
+schemas.py
+search/__init__.py
+search/semantic.py
+semantic.py
+tests/__init__.py
+tests/test_core.py
+tests/test_schema.py
+vector_store.py

HASH_MANIFEST.json

@@ -0,0 +1,62 @@
+{
+  "CODE_OF_CONDUCT.md": "b674f96cae26f0050be863c4b8782510fcae5ab855f0822ec4a0217763a84601",
+  "ETHICS.md": "d4f7c23c1e60297712786e392800158fcbe21116576496632e8221b0b8a16ff2",
+  "README.md": "e9bfdd2d6a4422fcb132bd4033a69d2241574c31fab71820e4643491b3b1225b",
+  "app.py": "c2a2b16ce45a327de0d42196104cb7fc50ec29ff1cb1fb95517a8ca655a3192a",
+  "appeal_pdf.py": "2d28ca1d0e796bfb5da25eac05a91354aadd58deefd041acded9a01a64055f9c",
+  "audit.py": "01c286d4067c6fffcb990391d8f750719c1ccac07eafc4477ccbdd1be4dd11e8",
+  "collaboration.py": "7cbd52c0da9be9f205b2901d8a94f28cb96612ffe506bcad1c7991885cd2d947",
+  "entity_graph.py": "dbe21fa0d8e7528daeee34d598efba836ab6370ad609de80746be1b12a4e0ff5",
+  "export_utils.py": "a01a088fd650a947a7831e795508208d3caa430d099aa5a8d7823ba462f0a80e",
+  "file_structure.txt": "6eee55e586751e3ae1405349f01dd35703e678d8e105ea19fc58eb15e4c2a6fa",
+  "foia_pdf.py": "babbd69a2da67681f15596ab254174310b8381d5853da72fe068d31d746725ab",
+  "foia_requests.py": "ca9c765bb7a591c462a94b0aa42957d1b3124128266d4880f0654895ce0ca6c0",
+  "gitattributes.txt": "11ad7efa24975ee4b0c3c3a38ed18737f0658a5f75a0a96787b576a78a023361",
+  "requirements.txt": "444bc9beedfa3fde82790f47c1e9b94bab90be2fefd0648de0ffdebbcc2eb61c",
+  "schemas.py": "e08b38513be2572af7d022e013f037c4f614f2117db85d4d776c408be96815ef",
+  "semantic.py": "4ffcf9149f08b8e69473e5418588dd370bbd470b137f2d0761901fccf09238cf",
+  "vector_store.py": "c61701e38e12150c541d284e13824341dde1794d3b4149d2a7d332b8023ad923",
+  "__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "README_PROD.md": "3b5d0a9f882f8f980a08452ca589a788b3c7cfe2ed8b7ca13a01f9c4a12e9060",
+  "adapters/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "adapters/cia.py": "a934e6a67aa6662391814036a9084779f15ad9ca5059f5461e2c374dfa9c3344",
+  "adapters/common.py": "c76c7ea1ce1616a2c99bfeec47ca046e75088f4d807f9c94ce5f87c9eeed5714",
+  "adapters/dea.py": "f1b9832aeaabecf5da8f1125e33883ce28e37d081149f6c75bf9ef49ac3ead8a",
+  "adapters/dhs.py": "66c44ee323135ee8e3c0cb7a2bb83d2d9dc20f7b88b6db4823e1bd5d03be6227",
+  "adapters/dia.py": "5c003321750582f502bcf0e2115956edb9af3aa8937917b72d7b25036b493f6d",
+  "adapters/dod.py": "410726bcab164fa9991d0ba61b3d9586d271ee4d55f65d1bd02193e84f02ed30",
+  "adapters/doj.py": "56080addcaef0a01d2395b6d44a93e9e271bc569a688f65657617d730a054eac",
+  "adapters/fbi.py": "b81b80972adf70b8283f2c16b241d17f46ab3ab73cd3ab4155dc88f7afbbcfc2",
+  "adapters/ice.py": "f0d06239d483933ba53966bc8015b9ca9f3ead3ebb535f4f963f5a26afd340b3",
+  "adapters/nia.py": "cbc240d23d7ac144d0ca0a49e83341df579903092c13c7603cfe438e7dd58a84",
+  "adapters/nsa.py": "a5a7ff4f8d3b1397bccc6095471de814aad75e2711566065f8cf7f4f43c59303",
+  "appeals/pdf_appeal.py": "cfe7ca493bf9a4280eff3d90494b2e2afc8bfed92ee99d5e175c1daf49ddadf6",
+  "appeals/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "collaboration/icij.py": "bd02217afd54664762594dfcd1e8088ac3666c641acd450d3b233cf05f08a641",
+  "collaboration/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "core/analysis.py": "e745cc6ad43d5193c92b5d7c417db4546ec301a96761090a319f7a477722dd99",
+  "core/appeals.py": "9ac66f34fdb2e741b6341de258291fd99db7f4a95862e39aa4cae94448726609",
+  "core/explain.py": "accdde04f5faf85b48302917f6274a12f06b9058fac5941cfa7ce9a64a6c45a3",
+  "core/index.py": "d266fc0aacbc2445b25cafbc29530e9138bb626090fb716681f300976927903c",
+  "core/multi_program.py": "444928c79f9778ebffcdb47262ba63b2eb19d2ed4d97d5632682a92e91861138",
+  "core/redaction.py": "b99bbbcb659e1f60902bca7e2bde5b0c28f371b7a6feb9daff489bb8fd96b878",
+  "core/search.py": "5843e5ee44d88688862b73e5457ff596dd229fc9433600c2e1a978868c8a2296",
+  "core/vector.py": "518e78f8c363735f5629584d2d5e25876a7f80063cd74e72a080723380141ce8",
+  "core/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "data/foia_sources.json": "8fe166a285717548afb937ad7a669020c60f91d9ec9f06dbdee9954f3396bd2a",
+  "data/demo/metadata.json": "89d069dd00b20d1c74eb6f192a09b9d11226d86d5f7754159b3b1717512302d3",
+  "data/demo/documents/aatip_sample.txt": "8b8d9a6167699a123885330093dac739025bfe0d7fabfdfd596707ab53db9f81",
+  "data/demo/documents/tencap_sample.txt": "e1930579e04e76cc2ced2b5b253fa59e907b28bbffba2f8c1710693cfc84b167",
+  "ingest/agency_registry.py": "89581ae5dcf6f0e5614939ce8538e17f4e22a1751d806bfce5cd51fbf9d35f85",
+  "ingest/cia_reading_room.py": "ebfa118842937a7929a1ce58998650f11081306e8a017d53f01e11262917f2e5",
+  "ingest/fbi_vault.py": "9a24fd572db556cc182239738ca2c551d6cb6a393a325f3fc8f6db9cbf1c157b",
+  "ingest/generic_public_foia.py": "60f174b9ada68330a70ca11898ae3fbb7d225e2f404265903a5079aaa274baa1",
+  "ingest/loader.py": "12b2b68d4c3a902270be73bebb1218314f19b225f7df4e436191f433378aca18",
+  "ingest/sources.py": "4b995bff081e14cbe3b66deb516abc74fce09e29f3e36463f60bbbcaf11b075b",
+  "ingest/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "search/semantic.py": "974faa592af9a67ec50a691180ad68d90e00d38244871680c0c45f31a77f8f36",
+  "search/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "tests/test_core.py": "58e5d87c0de8482328abcc27d7a1452cdc6a69740eb0de4395c78a250d12d79e",
+  "tests/test_schema.py": "04c0343db5c7516679395717a1dd4c2eca4e325cf038e5c6ee794c2a62649119",
+  "tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+}

HF_JUSTIFICATION.md

@@ -0,0 +1,12 @@
+
+This Hugging Face Space provides a public-interest federated search interface
+across U.S. Government FOIA Electronic Reading Rooms.
+
+Safeguards:
+- Public sources only
+- No authentication bypass
+- Rate limiting and health checks
+- Redaction-aware previews
+- Metadata indexing only
+
+Intended for journalism, research, and accountability.

HF_SPACE_README.md

@@ -0,0 +1,32 @@
+# FOIA Federated Document Search (Public Interest)
+
+🚀 **Hugging Face Space – Transparency & Accountability Tool**
+
+This application provides **semantic search across publicly released U.S. Government FOIA electronic reading rooms**.
+It does **not** access classified, private, or restricted systems.
+
+## What This Is
+- Federated FOIA document search
+- Semantic + keyword hybrid retrieval
+- Redaction-aware exports
+- Audit logging
+
+## What This Is NOT
+- Surveillance
+- Intelligence gathering
+- Law enforcement tooling
+- Political persuasion
+
+## Data Sources
+- CIA FOIA Electronic Reading Room
+- FBI Vault
+- Other agency FOIA libraries (public releases only)
+
+## Compliance
+- FOIA-only sources
+- robots.txt respected
+- Rate-limited adapters
+- Redaction before export
+
+## Intended Users
+Researchers, journalists, historians, and the general public.

LEGAL_MEMO.md

@@ -0,0 +1,6 @@
+
+FOIA Federated Search – Legal Summary
+
+This system indexes publicly released FOIA records.
+No restricted access, no scraping of protected systems.
+Fully compliant with 5 U.S.C. § 552.

README.md

@@ -0,0 +1,35 @@
+---
+title: FOIA Federated Search
+emoji: 📜
+colorFrom: blue
+colorTo: purple
+sdk: gradio
+sdk_version: "4.0"
+app_file: app.py
+pinned: false
+license: mit
+---
+
+# FOIA Federated Search (Public Interest)
+
+A Hugging Face Space that provides **live federated search** across publicly available
+U.S. Government FOIA Electronic Reading Rooms (CIA, FBI, DoD, and more).
+
+## Key Features
+- Live async fan-out search (no scraping beyond public endpoints)
+- Per-agency source toggles + result counts
+- Semantic *search-in-results* using FAISS + sentence-transformers
+- Local caching + deduplication
+- PDF export of search results
+- Inline document preview (where permitted by source)
+- Rate-limited, health-checked agency adapters
+
+## Trust & Safety
+- Queries only **public FOIA reading rooms**
+- Honors robots.txt, rate limits, and agency terms
+- No authentication bypass or restricted content
+- Designed for research, journalism, and public accountability
+
+## Legal
+All content remains hosted by the originating agency.
+This tool indexes metadata and snippets for discovery only.

README_PROD.md

@@ -0,0 +1,40 @@
+# FOIA HF Document Search — Production Build
+
+## Entry Point
+- `app.py` — orchestrates ingestion, semantic search, export, and audit hooks.
+
+## Ingestion Adapters (Present)
+- CIA Reading Room
+- FBI Vault
+- Generic Public FOIA
+
+## Missing / Stubbed Adapters (Recommended)
+- DoD (incl. components)
+- NSA
+- DIA
+- DHS
+- DEA
+- ICE
+
+## Vector Backend Assumptions
+- Current code supports abstract vector ops.
+- Recommended backends:
+  - FAISS (local)
+  - Chroma (persistent)
+  - HuggingFace embeddings
+  - OpenAI embeddings (optional)
+
+## Live Federated Search Upgrade
+- Async querying via `asyncio` + `httpx`
+- Adapter interface with rate limits
+- Response caching + deduplication
+- Circuit breakers for abuse prevention
+
+## Compliance
+- Respect robots.txt where applicable
+- Rate limiting per agency
+- Redaction before export
+- Audit logging enabled
+
+## Build Timestamp
+2026-01-09T23:51:16.728748Z

SOURCES.md

@@ -0,0 +1,23 @@
+# FOIA Public Sources
+
+All sources listed here are **public FOIA electronic reading rooms** or official public-release libraries.
+
+## Intelligence & Defense
+- CIA FOIA Electronic Reading Room — https://www.cia.gov/readingroom/
+- FBI Vault — https://vault.fbi.gov/
+- DARPA FOIA Library — https://www.darpa.mil/work-with-us/foia
+- NRO FOIA Reading Room — https://www.nro.gov/FOIA/
+- DoD FOIA Reading Room — https://www.esd.whs.mil/FOIA/Reading-Room/
+
+## Military Branches
+- U.S. Army FOIA — https://www.army.mil/foia
+- U.S. Navy FOIA — https://www.secnav.navy.mil/foia
+- U.S. Air Force FOIA — https://www.af.mil/FOIA/
+- U.S. Marine Corps FOIA — https://www.hqmc.marines.mil/Agencies/FOIA/
+- U.S. Space Force FOIA — https://www.spaceforce.mil/FOIA/
+- U.S. Coast Guard FOIA — https://www.uscg.mil/FOIA/
+
+## Other Agencies
+- DHS FOIA Library — https://www.dhs.gov/foia-library
+- DEA FOIA Reading Room — https://www.dea.gov/foia
+- Secret Service FOIA — https://www.secretservice.gov/foia

app.py

@@ -0,0 +1,51 @@
+import gradio as gr
+import asyncio
+from ingest.cia_reading_room import CIAAdapter
+from ingest.fbi_vault_live import FBIAdapter
+from ingest.dod_reading_room_live import DoDAdapter
+from core.async_search import fanout_search
+from core.cache import dedupe
+from core.cluster import cluster_results
+from core.citations import citation_block
+from core.redaction import redaction_confidence
+from core.journalist import journalist_export
+from core.explain import explain
+
+cia, fbi, dod = CIAAdapter(), FBIAdapter(), DoDAdapter()
+
+async def run(q):
+    res = await fanout_search([cia,fbi,dod], q)
+    return dedupe(res)
+
+with gr.Blocks() as demo:
+    gr.Markdown("# FOIA Federated Search — Supreme")
+
+    q = gr.Textbox(label="Query")
+    results_state = gr.State([])
+
+    with gr.Tabs():
+        with gr.Tab("Clustered Results"):
+            clusters = gr.JSON()
+        with gr.Tab("Citations"):
+            cites = gr.Markdown()
+        with gr.Tab("Explainability"):
+            explain_box = gr.JSON()
+
+    preview = gr.JSON(label="Redaction Confidence")
+
+    def _run(q):
+        res = asyncio.run(run(q))
+        cl = cluster_results(res)
+        cites_md = "\n".join(citation_block(r) for r in res[:5])
+        explain_data = explain(res)
+        red = {r.get("url"): redaction_confidence(r) for r in res}
+        return res, cl, cites_md, explain_data, red
+
+    btn = gr.Button("Search")
+    btn.click(_run, inputs=q, outputs=[results_state, clusters, cites, explain_box, preview])
+
+    exp = gr.Button("Journalist Export")
+    out = gr.File()
+    exp.click(lambda r: journalist_export(r, "/tmp/journalist_export.zip"), inputs=results_state, outputs=out)
+
+demo.launch()

appeal_pdf.py

@@ -0,0 +1,8 @@
+from reportlab.platypus import SimpleDocTemplate, Paragraph
+from reportlab.lib.styles import getSampleStyleSheet
+
+def generate_appeal_pdf(text, filename="appeal.pdf"):
+    doc = SimpleDocTemplate(filename)
+    styles = getSampleStyleSheet()
+    doc.build([Paragraph(text, styles["BodyText"])])
+    return filename

audit.py

@@ -0,0 +1,18 @@
+import uuid
+from datetime import datetime
+from typing import Dict, List
+
+_AUDIT_LOG: List[Dict] = []
+
+def log_event(action: str, payload: Dict) -> Dict:
+    entry = {
+        "id": str(uuid.uuid4()),
+        "timestamp": datetime.utcnow().isoformat() + "Z",
+        "action": action,
+        "payload": payload
+    }
+    _AUDIT_LOG.append(entry)
+    return entry
+
+def export_audit_log() -> List[Dict]:
+    return list(_AUDIT_LOG)

collaboration.py

@@ -0,0 +1,17 @@
+from datasets import Dataset
+from typing import Dict, List
+
+_COLLAB: List[Dict] = []
+
+def add_collaboration_note(document: str, note: str) -> Dict:
+    record = {
+        "document": document,
+        "note": note
+    }
+    _COLLAB.append(record)
+    return record
+
+def get_collaboration_dataset() -> Dataset:
+    if not _COLLAB:
+        return Dataset.from_dict({"document": [], "note": []})
+    return Dataset.from_list(_COLLAB)

entity_graph.py

@@ -0,0 +1,19 @@
+import networkx as nx
+from typing import List, Dict
+
+def build_entity_graph(docs: List[Dict]) -> Dict:
+    G = nx.Graph()
+
+    for d in docs:
+        agency = d.get("agency", "Unknown")
+        G.add_node(agency, group="agency")
+
+        for token in d.get("content", "").split():
+            if token.isupper() and len(token) > 2:
+                G.add_node(token, group="entity")
+                G.add_edge(agency, token)
+
+    return {
+        "nodes": [{"id": n, "group": G.nodes[n]["group"]} for n in G.nodes],
+        "links": [{"source": u, "target": v} for u, v in G.edges]
+    }

export_utils.py

@@ -0,0 +1,7 @@
+import json
+
+def export_json(data):
+    path = "/tmp/results.json"
+    with open(path, "w") as f:
+        json.dump(data, f, indent=2)
+    return path

file_structure.txt

@@ -0,0 +1,20 @@
+foia-chatbot/
+├── app.py
+├── requirements.txt
+│
+├── core/
+│   ├── search.py
+│   ├── analysis.py
+│   ├── vector.py
+│   ├── index.py
+│   ├── explain.py
+│   ├── multi_program.py
+│   ├── redaction.py
+│   └── appeals.py
+│
+├── data/
+│   └── demo/
+│       ├── metadata.json
+│       └── documents/
+│           ├── tencap_sample.txt
+│           └── aatip_sample.txt

foia_pdf.py

@@ -0,0 +1,70 @@
+# foia_pdf.py
+from reportlab.lib.pagesizes import LETTER
+from reportlab.pdfgen import canvas
+from datetime import datetime
+from typing import Dict
+import os
+import uuid
+
+OUTPUT_DIR = "generated_pdfs"
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+
+def generate_foia_appeal_pdf(record: Dict) -> str:
+    """
+    Generates a FOIA appeal draft PDF.
+    This does NOT submit anything to any agency.
+    """
+
+    filename = f"foia_appeal_{uuid.uuid4().hex}.pdf"
+    path = os.path.join(OUTPUT_DIR, filename)
+
+    c = canvas.Canvas(path, pagesize=LETTER)
+    width, height = LETTER
+
+    text = c.beginText(40, height - 50)
+    text.setFont("Times-Roman", 11)
+
+    text.textLine(f"FOIA Appeal Draft")
+    text.textLine("")
+    text.textLine(f"Date: {datetime.utcnow().strftime('%Y-%m-%d')}")
+    text.textLine("")
+    text.textLine(f"Agency: {record.get('agency')}")
+    text.textLine(f"Subject: {record.get('subject')}")
+    text.textLine("")
+    text.textLine("To Whom It May Concern,")
+    text.textLine("")
+    text.textLine(
+        "This letter serves as a formal appeal regarding the handling of a "
+        "Freedom of Information Act (FOIA) request."
+    )
+    text.textLine("")
+    text.textLine(
+        "The requested materials concern publicly released or previously "
+        "acknowledged records. Disclosure would contribute significantly "
+        "to public understanding of government operations."
+    )
+    text.textLine("")
+    text.textLine("Request Description:")
+    text.textLine(record.get("description", ""))
+    text.textLine("")
+    text.textLine(
+        "This appeal is submitted in good faith for journalistic, academic, "
+        "or public-interest review."
+    )
+    text.textLine("")
+    text.textLine("Sincerely,")
+    text.textLine("FOIA Declassified Document Search")
+    text.textLine("")
+    text.textLine("—")
+    text.textLine(
+        "Disclaimer: This document is a draft generated for reference only. "
+        "It does not constitute legal advice and does not submit a request "
+        "to any agency."
+    )
+
+    c.drawText(text)
+    c.showPage()
+    c.save()
+
+    return path

foia_requests.py

@@ -0,0 +1,52 @@
+from typing import Dict, List
+import json
+import os
+import uuid
+from datetime import datetime
+
+FOIA_STORE = "data/foia_requests.json"
+
+
+def _load_requests() -> List[Dict]:
+    if not os.path.exists(FOIA_STORE):
+        return []
+    try:
+        with open(FOIA_STORE, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except Exception:
+        return []
+
+
+def _save_requests(requests: List[Dict]) -> None:
+    os.makedirs(os.path.dirname(FOIA_STORE), exist_ok=True)
+    with open(FOIA_STORE, "w", encoding="utf-8") as f:
+        json.dump(requests, f, indent=2)
+
+
+def add_foia_request(
+    agency: str,
+    subject: str,
+    description: str,
+    requester_type: str = "Journalist"
+) -> Dict:
+    """
+    Store a FOIA request record (tracking only).
+    No submission to agencies is performed.
+    """
+
+    record = {
+        "id": str(uuid.uuid4()),
+        "timestamp": datetime.utcnow().isoformat() + "Z",
+        "agency": agency,
+        "subject": subject,
+        "description": description,
+        "requester_type": requester_type,
+        "status": "Draft",
+        "notes": "Generated by FOIA Declassified Document Search (tracking only)"
+    }
+
+    requests = _load_requests()
+    requests.append(record)
+    _save_requests(requests)
+
+    return record

gitattributes.txt

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

requirements.txt

@@ -0,0 +1,11 @@
+beautifulsoup4
+datasets
+faiss-cpu
+gradio
+gradio>=4.44.0
+httpx
+networkx
+pandas
+requests
+sentence-transformers
+uuid

schemas.py

@@ -0,0 +1,8 @@
+from typing import List, Dict, Any
+
+def validate_results(results: List[Dict[str, Any]]) -> None:
+    for r in results:
+        assert isinstance(r["document"], str)
+        assert isinstance(r["agency"], str)
+        assert isinstance(r["year"], int)
+        assert isinstance(r["excerpt"], str)

semantic.py

@@ -0,0 +1,18 @@
+import faiss
+import numpy as np
+from sentence_transformers import SentenceTransformer
+from typing import List, Dict
+
+model = SentenceTransformer("all-MiniLM-L6-v2")
+
+def build_faiss_index(docs: List[Dict]):
+    texts = [d["content"] for d in docs]
+    embeddings = model.encode(texts)
+    index = faiss.IndexFlatL2(embeddings.shape[1])
+    index.add(np.array(embeddings))
+    return index, embeddings
+
+def semantic_search(query: str, docs: List[Dict], index):
+    q_emb = model.encode([query])
+    D, I = index.search(np.array(q_emb), k=5)
+    return [docs[i] for i in I[0]]

space.yaml

@@ -0,0 +1,8 @@
+title: FOIA Federated Search
+emoji: 📄
+colorFrom: blue
+colorTo: gray
+sdk: gradio
+python_version: "3.10"
+app_file: app.py
+pinned: false

trust_safety.md

@@ -0,0 +1,12 @@
+# Trust & Safety Statement
+
+This Space indexes only publicly released FOIA documents.
+No private data is collected or stored.
+
+Safeguards:
+- Rate limiting
+- Redaction pipeline
+- Audit logging
+- Explicit prohibition on misuse
+
+This tool exists solely to enhance public access to already-released government records.

vector_store.py

@@ -0,0 +1,17 @@
+import faiss
+import numpy as np
+
+# demo embeddings
+INDEX = faiss.IndexFlatL2(384)
+DOCS = []
+
+def semantic_search(query: str):
+    # placeholder demo
+    return [
+        {
+            "document": "UAP Analysis Memo",
+            "agency": "DIA",
+            "year": 2010,
+            "excerpt": "Unidentified aerial phenomena..."
+        }
+    ]