Update app.py

app.py

@@ -1,56 +1,47 @@
 # ======================================================
 # Federal FOIA Intelligence Search
-# Court-Safe / HF-Reviewer-Safe Reference Implementation
+# Hugging Face Spaces – Reviewer Safe
 # ======================================================
 
-import os, io, zipfile, tempfile, hashlib, base64
+import os
+import io
+import zipfile
+import hashlib
 from datetime import datetime
 from urllib.parse import quote_plus
-import requests
 
 import gradio as gr
 from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
-from fastapi.responses import JSONResponse
-
-# ======================================================
-# OPTIONAL PDF SUPPORT
-# ======================================================
-
-PDF_THUMBNAILS_AVAILABLE = False
-try:
-    from pdf2image import convert_from_bytes
-    PDF_THUMBNAILS_AVAILABLE = True
-except Exception:
-    pass
 
 # ======================================================
 # GOVERNANCE FLAGS (HARD)
 # ======================================================
 
 ENABLE_AI = True
-ENABLE_FAISS_PHASE_4 = False          # FORMAL APPROVAL REQUIRED
-ENABLE_DOC_LEVEL_APIS = False         # CIA/FBI DO NOT PROVIDE
+ENABLE_DOC_LEVEL_APIS = False   # No public CIA/FBI doc APIs exist
+ENABLE_FAISS_PHASE_4 = False   # Requires formal approval
 
 # ======================================================
-# SESSION STATE (EPHEMERAL)
+# SESSION STATE
 # ======================================================
 
 LAST_RESULTS = []
 AI_APPENDIX = None
 
 # ======================================================
-# CRYPTOGRAPHY
+# CRYPTOGRAPHIC PROVENANCE
 # ======================================================
 
-def sha256_text(t: str) -> str:
-    return hashlib.sha256(t.encode()).hexdigest()
+def sha256_text(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
 
 def provenance_block(payload: str, ai=False) -> str:
     return "\n".join([
-        "Tool-Version: 2.1.0",
+        "Tool: Federal FOIA Intelligence Search",
+        "Version: 1.9.0",
         f"Generated-UTC: {datetime.utcnow().isoformat()}",
-        f"Content-SHA256: {sha256_text(payload)}",
+        f"SHA256: {sha256_text(payload)}",
         "Public-Source-Only: true",
         f"AI-Assisted: {'true' if ai else 'false'}",
         "Court-Safe: true",
@@ -61,71 +52,40 @@ def provenance_block(payload: str, ai=False) -> str:
 # ======================================================
 
 class FOIAAdapter:
-    agency = "UNKNOWN"
+    agency = ""
     search_url = ""
 
     def search(self, query):
-        url = self.search_url.format(q=quote_plus(query))
         return [{
             "agency": self.agency,
             "title": f"{self.agency} FOIA Reading Room",
-            "resolved_url": url,
-            "timestamp": datetime.utcnow().isoformat(),
-            "is_pdf": False,
-            "thumbnails": [],
+            "url": self.search_url.format(q=quote_plus(query)),
+            "why": "This is a public FOIA electronic reading room result."
         }]
 
-class CIA(FOIAAdapter):
+class CIAAdapter(FOIAAdapter):
     agency = "CIA"
     search_url = "https://www.cia.gov/readingroom/search/site/{q}"
 
-class FBI(FOIAAdapter):
+class FBIAdapter(FOIAAdapter):
     agency = "FBI"
     search_url = "https://vault.fbi.gov/search?SearchableText={q}"
 
-class DOJ(FOIAAdapter):
+class DOJAdapter(FOIAAdapter):
     agency = "DOJ"
-    search_url = "https://www.justice.gov/foia/library?search={q}"
+    search_url = "https://www.justice.gov/oip/foia-library/search?search_api_fulltext={q}"
 
-class DHS(FOIAAdapter):
+class DHSAdapter(FOIAAdapter):
     agency = "DHS"
-    search_url = "https://www.dhs.gov/foia-library/search?search={q}"
+    search_url = "https://www.dhs.gov/foia-library/search?search_api_fulltext={q}"
 
-ALL_ADAPTERS = {
-    "CIA": CIA(),
-    "FBI": FBI(),
-    "DOJ": DOJ(),
-    "DHS": DHS(),
+ADAPTERS = {
+    "CIA": CIAAdapter(),
+    "FBI": FBIAdapter(),
+    "DOJ": DOJAdapter(),
+    "DHS": DHSAdapter(),
 }
 
-# ======================================================
-# PDF DETECTION (SAFE)
-# ======================================================
-
-def resolve_pdf(url):
-    try:
-        r = requests.get(url, timeout=10, allow_redirects=True)
-        ct = r.headers.get("content-type", "").lower()
-        is_pdf = r.url.lower().endswith(".pdf") or "application/pdf" in ct
-        return is_pdf, r.url
-    except Exception:
-        return False, url
-
-def generate_thumbnails(url, pages=2):
-    if not PDF_THUMBNAILS_AVAILABLE:
-        return []
-    try:
-        r = requests.get(url, timeout=10)
-        images = convert_from_bytes(r.content, first_page=1, last_page=pages)
-        thumbs = []
-        for img in images:
-            buf = io.BytesIO()
-            img.save(buf, format="PNG")
-            thumbs.append(base64.b64encode(buf.getvalue()).decode())
-        return thumbs
-    except Exception:
-        return []
-
 # ======================================================
 # SEARCH
 # ======================================================
@@ -135,150 +95,193 @@ def run_search(query, agencies):
     LAST_RESULTS = []
 
     rows = []
-    for name in agencies:
-        for r in ALL_ADAPTERS[name].search(query):
-            is_pdf, resolved = resolve_pdf(r["resolved_url"])
-            r["resolved_url"] = resolved
-            r["is_pdf"] = is_pdf
-            r["thumbnails"] = generate_thumbnails(resolved) if is_pdf else []
-            r["hash"] = sha256_text(resolved)[:16]
+    for agency in agencies:
+        for r in ADAPTERS[agency].search(query):
+            r["hash"] = sha256_text(r["url"])[:16]
             LAST_RESULTS.append(r)
-            rows.append([r["agency"], r["title"], resolved, r["hash"]])
+            rows.append([
+                r["agency"],
+                r["title"],
+                r["url"],
+                r["hash"],
+            ])
 
     return rows, render_cards(), "Search complete."
 
 # ======================================================
-# ASK-AI (OPT-IN)
+# AI (STRICT USER INITIATION)
 # ======================================================
 
 def ask_ai(index: int):
     global AI_APPENDIX
-    r = LAST_RESULTS[index]
-
-    if not (ENABLE_AI and r["is_pdf"]):
-        return "AI assistance is unavailable for this result."
 
+    r = LAST_RESULTS[index]
     text = (
         "AI Assistive Summary (Non-Authoritative)\n\n"
         f"Agency: {r['agency']}\n"
-        f"Source: {r['resolved_url']}\n\n"
-        "This assists review of a public FOIA document only."
+        f"Source URL: {r['url']}\n\n"
+        "This summary assists review of a public FOIA document."
     )
 
     AI_APPENDIX = {
         "text": text,
-        "hash": sha256_text(text),
-        "prov": provenance_block(text, ai=True),
+        "prov": provenance_block(text, ai=True)
     }
 
     return text + "\n\n" + AI_APPENDIX["prov"]
 
 # ======================================================
-# COURT BUNDLE
+# COURT / CM-ECF BUNDLE
 # ======================================================
 
 def generate_court_bundle():
-    with tempfile.TemporaryDirectory() as td:
-        path = os.path.join(td, "court_bundle.zip")
-        with zipfile.ZipFile(path, "w") as z:
-            for i, r in enumerate(LAST_RESULTS, 1):
-                body = (
-                    f"{r['agency']} FOIA Reading Room\n"
-                    f"{r['resolved_url']}\n\n"
-                    + provenance_block(r["resolved_url"])
-                )
-                z.writestr(f"Exhibit_{i:03d}.txt", body)
-
-            if AI_APPENDIX:
-                z.writestr("Exhibit_AI_Appendix.txt", AI_APPENDIX["text"])
-                z.writestr("Exhibit_AI_Appendix.provenance.txt", AI_APPENDIX["prov"])
-
-        return path
+    buf = io.BytesIO()
+    with zipfile.ZipFile(buf, "w") as z:
+        for i, r in enumerate(LAST_RESULTS, 1):
+            body = (
+                f"{r['agency']} FOIA Reading Room\n"
+                f"{r['url']}\n\n"
+                + provenance_block(r["url"])
+            )
+            z.writestr(f"Exhibit_{i:03d}.txt", body)
+
+        if AI_APPENDIX:
+            z.writestr("Exhibit_AI_Appendix.txt", AI_APPENDIX["text"])
+            z.writestr("Exhibit_AI_Appendix.provenance.txt", AI_APPENDIX["prov"])
+
+        z.writestr("Judicial_Notice.txt", JUDICIAL_NOTICE)
+        z.writestr("HF_Reviewer_Explanation.txt", HF_REVIEWER_EXPLANATION)
+
+    buf.seek(0)
+    return buf
 
 # ======================================================
-# GOVERNANCE SITE LOADING
+# STATIC GOVERNANCE TEXT
 # ======================================================
 
-def load_governance_md():
-    path = "governance-site/index.md"
-    if os.path.exists(path):
-        with open(path, "r", encoding="utf-8") as f:
-            return f.read()
-    return "_Governance documentation not found._"
+HF_REVIEWER_EXPLANATION = """
+This application performs federated link-out searches
+to public FOIA electronic reading rooms.
+
+No scraping, crawling, authentication bypass,
+or automated document ingestion occurs.
+
+CIA and FBI search behavior mirrors public web interfaces.
+"""
+
+JUDICIAL_NOTICE = """
+Judicial Notice Regarding FOIA Navigation
+
+This tool does not retrieve or alter records.
+It provides navigational references to public
+government FOIA reading rooms.
+
+Any linked document remains hosted
+and controlled by the originating agency.
+"""
 
 # ======================================================
-# FASTAPI
+# FASTAPI (STATIC GOVERNANCE SITE)
 # ======================================================
 
 api = FastAPI()
 
-@api.get("/ask_ai")
-def ask_ai_api(index: int):
-    return JSONResponse({"result": ask_ai(index)})
-
 if os.path.exists("governance-site"):
-    api.mount("/gov", StaticFiles(directory="governance-site", html=True))
+    api.mount(
+        "/governance",
+        StaticFiles(directory="governance-site", html=True),
+        name="governance",
+    )
 
 # ======================================================
-# UI
+# UI HELPERS
 # ======================================================
 
-CSS = """
-.tab-nav { position: sticky; top: 0; z-index: 999; background: #0f0f0f; }
-@media (max-width: 768px) {
-  .desktop-only { display: none !important; }
-}
-button { border-radius: 999px !important; }
-"""
-
 def render_cards():
     cards = []
     for i, r in enumerate(LAST_RESULTS):
-        download_html = (
-            f'<a href="{r["resolved_url"]}" download>Download</a>'
-            if r["is_pdf"]
-            else '<span style="opacity:.4">Download</span>'
-        )
-
         cards.append(f"""
-        <div style="border:1px solid #333;border-radius:18px;padding:16px;margin-bottom:16px;">
-          <b>{r['agency']}</b><br/>
-          {r['title']}<br/>
-          <div style="margin-top:10px;">
-            <a href="{r['resolved_url']}" target="_blank">View</a> |
-            {download_html} |
-            <a href="{r['resolved_url']}" target="_blank">Share</a>
-            <button style="background:#1e88e5;color:white;padding:4px 14px;margin-left:10px;border:none;"
-              onclick="fetch('/ask_ai?index={i}')">
-              Ask AI
-            </button>
-          </div>
-          <div style="font-size:.75em;color:#777;margin-top:6px;">
-            Why am I seeing this? This is a public FOIA reading-room result.
-          </div>
+        <div class="card">
+            <b>{r['agency']}</b><br/>
+            {r['title']}<br/>
+            <div class="links">
+              <a href="{r['url']}" target="_blank">View</a> |
+              <span class="disabled">Download</span> |
+              <span class="disabled">Share</span>
+              <button class="ask-ai" onclick="fetch('/ask_ai?index={i}')">
+                Ask AI
+              </button>
+            </div>
+            <div class="why">
+              Why am I seeing this? {r['why']}
+            </div>
         </div>
         """)
     return "".join(cards)
 
-with gr.Blocks(css=CSS) as demo:
+# ======================================================
+# STYLES
+# ======================================================
+
+CSS = """
+.tabs { position: sticky; top: 0; background: #0b0f17; z-index: 10; }
+button.primary, .ask-ai {
+  background: #1e88e5 !important;
+  border-radius: 999px !important;
+  color: white !important;
+}
+.card {
+  border: 1px solid #333;
+  border-radius: 16px;
+  padding: 14px;
+  margin-bottom: 14px;
+}
+.disabled { color: #666; cursor: not-allowed; }
+.why { font-size: 0.8em; color: #aaa; margin-top: 6px; }
+"""
+
+# ======================================================
+# UI
+# ======================================================
+
+with gr.Blocks(css=CSS) as app:
     gr.Markdown("## Federal FOIA Intelligence Search")
 
-    with gr.Tab("Search"):
-        agencies = gr.CheckboxGroup(list(ALL_ADAPTERS.keys()), value=list(ALL_ADAPTERS.keys()))
-        query = gr.Textbox(placeholder="Search FOIA reading rooms")
-        table = gr.Dataframe(
-            headers=["Agency", "Title", "URL", "Hash"],
-            elem_classes=["desktop-only"],
-        )
-        cards = gr.HTML()
-        status = gr.Textbox()
-        gr.Button("Search").click(run_search, [query, agencies], [table, cards, status])
-
-    with gr.Tab("Court / Clerk"):
-        gr.Button("Generate Court Bundle").click(lambda: generate_court_bundle(), None, gr.File())
-
-    with gr.Tab("Governance"):
-        gr.Markdown(load_governance_md())
-
-demo.queue()
-demo.launch(server_name="0.0.0.0", server_port=7860)
+    with gr.Tabs():
+        with gr.Tab("Search"):
+            agencies = gr.CheckboxGroup(
+                list(ADAPTERS.keys()),
+                value=list(ADAPTERS.keys())
+            )
+            query = gr.Textbox(placeholder="Search FOIA reading rooms")
+            table = gr.Dataframe(
+                headers=["Agency", "Title", "URL", "Hash"]
+            )
+            cards = gr.HTML()
+            status = gr.Textbox()
+
+            gr.Button("Search", elem_classes=["primary"]).click(
+                run_search,
+                [query, agencies],
+                [table, cards, status]
+            )
+
+        with gr.Tab("Court / Clerk"):
+            gr.Button("Generate Court Bundle").click(
+                generate_court_bundle,
+                None,
+                gr.File()
+            )
+
+        with gr.Tab("Governance"):
+            gr.Markdown(
+                "### Governance & Transparency\n\n"
+                "[Open Governance Site](/governance/index.html)"
+            )
+
+# ======================================================
+# LAUNCH (HF SAFE)
+# ======================================================
+
+app.queue()
+app.launch(server_name="0.0.0.0", server_port=7860)