Update app.py

app.py

@@ -1,90 +1,127 @@
 # ======================================================
 # Federal FOIA Intelligence Search
-# Hugging Face Spaces – Reviewer Safe
+# HF Reviewer–Safe / Court-Safe LIVE IMPLEMENTATION
 # ======================================================
 
-import os
-import io
-import zipfile
-import hashlib
+import os, io, time, zipfile, tempfile, hashlib, base64
 from datetime import datetime
 from urllib.parse import quote_plus
+import requests
 
 import gradio as gr
 from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
+from fastapi.responses import JSONResponse
 
 # ======================================================
-# GOVERNANCE FLAGS (HARD)
+# GOVERNANCE FLAGS (NON-NEGOTIABLE)
 # ======================================================
 
-ENABLE_AI = True
-ENABLE_DOC_LEVEL_APIS = False   # No public CIA/FBI doc APIs exist
-ENABLE_FAISS_PHASE_4 = False   # Requires formal approval
+ENABLE_AI = True                  # user-initiated only
+ENABLE_FAISS_PHASE_4 = False      # hard disabled
+ENABLE_DOC_LEVEL_APIS = False     # CIA/FBI have no public APIs (documented)
 
 # ======================================================
-# SESSION STATE
+# OPTIONAL PDF SUPPORT
+# ======================================================
+
+PDF_THUMBNAILS_AVAILABLE = False
+PDF_TEXT_AVAILABLE = False
+
+try:
+    from pdf2image import convert_from_bytes
+    PDF_THUMBNAILS_AVAILABLE = True
+except Exception:
+    pass
+
+try:
+    from pdfminer.high_level import extract_text
+    PDF_TEXT_AVAILABLE = True
+except Exception:
+    pass
+
+# ======================================================
+# SESSION STATE (EPHEMERAL)
 # ======================================================
 
 LAST_RESULTS = []
 AI_APPENDIX = None
 
 # ======================================================
-# CRYPTOGRAPHIC PROVENANCE
+# CRYPTOGRAPHY / PROVENANCE
 # ======================================================
 
-def sha256_text(text: str) -> str:
-    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+def sha256_text(t: str) -> str:
+    return hashlib.sha256(t.encode()).hexdigest()
 
 def provenance_block(payload: str, ai=False) -> str:
     return "\n".join([
-        "Tool: Federal FOIA Intelligence Search",
-        "Version: 1.9.0",
+        "Tool-Version: 1.9.0",
         f"Generated-UTC: {datetime.utcnow().isoformat()}",
-        f"SHA256: {sha256_text(payload)}",
+        f"Content-SHA256: {sha256_text(payload)}",
         "Public-Source-Only: true",
         f"AI-Assisted: {'true' if ai else 'false'}",
         "Court-Safe: true",
     ])
 
 # ======================================================
-# FOIA ADAPTERS (LINK-OUT ONLY)
+# FOIA ADAPTERS (LINK-OUT ONLY — CORRECT SEARCH EMBEDDING)
 # ======================================================
 
 class FOIAAdapter:
-    agency = ""
+    agency = "UNKNOWN"
     search_url = ""
 
     def search(self, query):
+        url = self.search_url.format(q=quote_plus(query))
         return [{
             "agency": self.agency,
-            "title": f"{self.agency} FOIA Reading Room",
-            "url": self.search_url.format(q=quote_plus(query)),
-            "why": "This is a public FOIA electronic reading room result."
+            "title": f"{self.agency} FOIA Reading Room Results",
+            "resolved_url": url,
+            "timestamp": datetime.utcnow().isoformat(),
         }]
 
-class CIAAdapter(FOIAAdapter):
+class CIA(FOIAAdapter):
     agency = "CIA"
+    # CIA requires /search/site/{query}
     search_url = "https://www.cia.gov/readingroom/search/site/{q}"
 
-class FBIAdapter(FOIAAdapter):
+class FBI(FOIAAdapter):
     agency = "FBI"
     search_url = "https://vault.fbi.gov/search?SearchableText={q}"
 
-class DOJAdapter(FOIAAdapter):
-    agency = "DOJ"
-    search_url = "https://www.justice.gov/oip/foia-library/search?search_api_fulltext={q}"
+ALL_ADAPTERS = {
+    "CIA": CIA(),
+    "FBI": FBI(),
+}
 
-class DHSAdapter(FOIAAdapter):
-    agency = "DHS"
-    search_url = "https://www.dhs.gov/foia-library/search?search_api_fulltext={q}"
+# ======================================================
+# PDF RESOLUTION + THUMBNAILS
+# ======================================================
 
-ADAPTERS = {
-    "CIA": CIAAdapter(),
-    "FBI": FBIAdapter(),
-    "DOJ": DOJAdapter(),
-    "DHS": DHSAdapter(),
-}
+def resolve_pdf(url):
+    try:
+        r = requests.get(url, timeout=10, allow_redirects=True)
+        ct = r.headers.get("content-type", "").lower()
+        is_pdf = r.url.lower().endswith(".pdf") or "application/pdf" in ct
+        return is_pdf, r.url
+    except Exception:
+        return False, url
+
+def pdf_thumbnails(url, pages=2):
+    if not PDF_THUMBNAILS_AVAILABLE:
+        return []
+    try:
+        r = requests.get(url, timeout=10)
+        images = convert_from_bytes(r.content, first_page=1, last_page=pages)
+        thumbs = []
+        for img in images:
+            buf = io.BytesIO()
+            img.save(buf, format="PNG")
+            thumbs.append(base64.b64encode(buf.getvalue()).decode())
+        return thumbs
+    except Exception:
+        return []
 
 # ======================================================
 # SEARCH
@@ -93,38 +130,47 @@ ADAPTERS = {
 def run_search(query, agencies):
     global LAST_RESULTS
     LAST_RESULTS = []
-
     rows = []
-    for agency in agencies:
-        for r in ADAPTERS[agency].search(query):
-            r["hash"] = sha256_text(r["url"])[:16]
+
+    for name in agencies:
+        for r in ALL_ADAPTERS[name].search(query):
+            is_pdf, resolved = resolve_pdf(r["resolved_url"])
+            r["is_pdf"] = is_pdf
+            r["resolved_url"] = resolved
+            r["hash"] = sha256_text(resolved)[:16]
+            r["thumbnails"] = pdf_thumbnails(resolved) if is_pdf else []
             LAST_RESULTS.append(r)
+
             rows.append([
                 r["agency"],
                 r["title"],
-                r["url"],
+                r["resolved_url"],
                 r["hash"],
             ])
 
     return rows, render_cards(), "Search complete."
 
 # ======================================================
-# AI (STRICT USER INITIATION)
+# ASK-AI (STRICTLY GATED)
 # ======================================================
 
 def ask_ai(index: int):
     global AI_APPENDIX
-
     r = LAST_RESULTS[index]
+
+    if not (ENABLE_AI and r["is_pdf"]):
+        return "AI is disabled for this result."
+
     text = (
         "AI Assistive Summary (Non-Authoritative)\n\n"
         f"Agency: {r['agency']}\n"
-        f"Source URL: {r['url']}\n\n"
-        "This summary assists review of a public FOIA document."
+        f"URL: {r['resolved_url']}\n\n"
+        "This output assists review of a publicly released FOIA document."
     )
 
     AI_APPENDIX = {
         "text": text,
+        "hash": sha256_text(text),
         "prov": provenance_block(text, ai=True)
     }
 
@@ -135,57 +181,33 @@ def ask_ai(index: int):
 # ======================================================
 
 def generate_court_bundle():
-    buf = io.BytesIO()
-    with zipfile.ZipFile(buf, "w") as z:
-        for i, r in enumerate(LAST_RESULTS, 1):
-            body = (
-                f"{r['agency']} FOIA Reading Room\n"
-                f"{r['url']}\n\n"
-                + provenance_block(r["url"])
-            )
-            z.writestr(f"Exhibit_{i:03d}.txt", body)
-
-        if AI_APPENDIX:
-            z.writestr("Exhibit_AI_Appendix.txt", AI_APPENDIX["text"])
-            z.writestr("Exhibit_AI_Appendix.provenance.txt", AI_APPENDIX["prov"])
-
-        z.writestr("Judicial_Notice.txt", JUDICIAL_NOTICE)
-        z.writestr("HF_Reviewer_Explanation.txt", HF_REVIEWER_EXPLANATION)
-
-    buf.seek(0)
-    return buf
+    with tempfile.TemporaryDirectory() as td:
+        path = os.path.join(td, "court_bundle.zip")
+        with zipfile.ZipFile(path, "w") as z:
+            for i, r in enumerate(LAST_RESULTS, 1):
+                body = (
+                    f"{r['agency']} FOIA Reading Room\n"
+                    f"{r['resolved_url']}\n\n"
+                    + provenance_block(r["resolved_url"])
+                )
+                z.writestr(f"Exhibit_{i:03d}.txt", body)
+
+            if AI_APPENDIX:
+                z.writestr("Exhibit_AI.txt", AI_APPENDIX["text"])
+                z.writestr("Exhibit_AI.provenance.txt", AI_APPENDIX["prov"])
+
+        return path
 
 # ======================================================
-# STATIC GOVERNANCE TEXT
-# ======================================================
-
-HF_REVIEWER_EXPLANATION = """
-This application performs federated link-out searches
-to public FOIA electronic reading rooms.
-
-No scraping, crawling, authentication bypass,
-or automated document ingestion occurs.
-
-CIA and FBI search behavior mirrors public web interfaces.
-"""
-
-JUDICIAL_NOTICE = """
-Judicial Notice Regarding FOIA Navigation
-
-This tool does not retrieve or alter records.
-It provides navigational references to public
-government FOIA reading rooms.
-
-Any linked document remains hosted
-and controlled by the originating agency.
-"""
-
-# ======================================================
-# FASTAPI (STATIC GOVERNANCE SITE)
+# FASTAPI (FOR ASK-AI + GOVERNANCE)
 # ======================================================
 
 api = FastAPI()
 
+@api.get("/ask_ai")
+def ask_ai_api(index: int):
+    return JSONResponse({"result": ask_ai(index)})
+
 if os.path.exists("governance-site"):
     api.mount(
         "/governance",
@@ -194,94 +216,86 @@ if os.path.exists("governance-site"):
     )
 
 # ======================================================
-# UI HELPERS
+# UI RENDERING
 # ======================================================
 
 def render_cards():
     cards = []
     for i, r in enumerate(LAST_RESULTS):
+        thumbs = "".join(
+            f'<img src="data:image/png;base64,{t}" '
+            f'style="width:120px;border-radius:8px;margin-right:6px;" />'
+            for t in r["thumbnails"]
+        )
+
         cards.append(f"""
         <div class="card">
-            <b>{r['agency']}</b><br/>
-            {r['title']}<br/>
-            <div class="links">
-              <a href="{r['url']}" target="_blank">View</a> |
-              <span class="disabled">Download</span> |
-              <span class="disabled">Share</span>
-              <button class="ask-ai" onclick="fetch('/ask_ai?index={i}')">
-                Ask AI
-              </button>
-            </div>
-            <div class="why">
-              Why am I seeing this? {r['why']}
-            </div>
+          <div class="header">
+            <b>{r['agency']}</b>
+            <button class="ask-ai" onclick="fetch('/ask_ai?index={i}')">
+              Ask AI
+            </button>
+          </div>
+
+          <div class="title">{r['title']}</div>
+
+          {thumbs}
+
+          <div class="links">
+            <a href="{r['resolved_url']}" target="_blank">View</a>
+            {" | <a href='"+r['resolved_url']+"' download>Download</a>" if r["is_pdf"] else ""}
+            {" | <a href='"+r['resolved_url']+"' target='_blank'>Share</a>"}
+          </div>
+
+          <div class="why">
+            Why am I seeing this?  
+            This links to a publicly released FOIA reading-room result.
+          </div>
         </div>
         """)
+
     return "".join(cards)
 
 # ======================================================
-# STYLES
+# GRADIO UI (NO DEPRECATED ARGS)
 # ======================================================
 
 CSS = """
-.tabs { position: sticky; top: 0; background: #0b0f17; z-index: 10; }
-button.primary, .ask-ai {
-  background: #1e88e5 !important;
-  border-radius: 999px !important;
-  color: white !important;
-}
-.card {
-  border: 1px solid #333;
-  border-radius: 16px;
-  padding: 14px;
-  margin-bottom: 14px;
-}
-.disabled { color: #666; cursor: not-allowed; }
-.why { font-size: 0.8em; color: #aaa; margin-top: 6px; }
+.card { border:1px solid #2a2a2a; border-radius:18px; padding:16px; margin-bottom:18px; background:#0f0f0f; }
+.header { display:flex; justify-content:space-between; align-items:center; }
+.ask-ai { background:#1e88e5; color:white; border:none; border-radius:999px; padding:6px 14px; font-weight:600; }
+.links a { color:#64b5f6; text-decoration:none; }
+.why { font-size:0.75rem; color:#aaa; margin-top:6px; }
 """
 
-# ======================================================
-# UI
-# ======================================================
-
-with gr.Blocks(css=CSS) as app:
+with gr.Blocks(css=CSS) as demo:
     gr.Markdown("## Federal FOIA Intelligence Search")
 
-    with gr.Tabs():
-        with gr.Tab("Search"):
-            agencies = gr.CheckboxGroup(
-                list(ADAPTERS.keys()),
-                value=list(ADAPTERS.keys())
-            )
-            query = gr.Textbox(placeholder="Search FOIA reading rooms")
-            table = gr.Dataframe(
-                headers=["Agency", "Title", "URL", "Hash"]
-            )
-            cards = gr.HTML()
-            status = gr.Textbox()
-
-            gr.Button("Search", elem_classes=["primary"]).click(
-                run_search,
-                [query, agencies],
-                [table, cards, status]
-            )
-
-        with gr.Tab("Court / Clerk"):
-            gr.Button("Generate Court Bundle").click(
-                generate_court_bundle,
-                None,
-                gr.File()
-            )
-
-        with gr.Tab("Governance"):
-            gr.Markdown(
-                "### Governance & Transparency\n\n"
-                "[Open Governance Site](/governance/index.html)"
-            )
+    with gr.Tab("Search"):
+        agencies = gr.CheckboxGroup(list(ALL_ADAPTERS.keys()), value=list(ALL_ADAPTERS.keys()))
+        query = gr.Textbox(placeholder="Search FOIA reading rooms")
+        table = gr.Dataframe(headers=["Agency", "Title", "URL", "Hash"])
+        cards = gr.HTML()
+        status = gr.Textbox()
+        gr.Button("Search", variant="primary").click(
+            run_search,
+            [query, agencies],
+            [table, cards, status]
+        )
+
+    with gr.Tab("Court / Clerk"):
+        gr.Button("Generate CM/ECF Bundle").click(
+            lambda: generate_court_bundle(),
+            None,
+            gr.File()
+        )
+
+    with gr.Tab("Governance"):
+        gr.HTML('<iframe src="/governance/index.html" style="width:100%;height:700px;border:none;"></iframe>')
 
 # ======================================================
-# LAUNCH (HF SAFE)
+# START (HF-SAFE)
 # ======================================================
 
-app.queue()
-app.launch(server_name="0.0.0.0", server_port=7860)
+demo.queue()
+demo.launch(server_name="0.0.0.0", server_port=7860)