Update app.py

app.py

@@ -3,61 +3,86 @@
 # HF Reviewer–Safe Reference Implementation
 # ======================================================
 
-import gradio as gr
-import time, hashlib, zipfile, os, tempfile
+import time, hashlib, io, zipfile, os, tempfile, base64
 from datetime import datetime
 from urllib.parse import quote_plus
-
 import requests
-from fastapi import FastAPI, Query
+
+import gradio as gr
+from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
-from fastapi.responses import JSONResponse, FileResponse
+from fastapi.responses import JSONResponse
+
+from reportlab.platypus import SimpleDocTemplate, Paragraph, PageBreak
+from reportlab.lib.styles import getSampleStyleSheet
+from reportlab.lib.pagesizes import LETTER
 
 # ======================================================
 # HARD GOVERNANCE FLAGS (NON-NEGOTIABLE)
 # ======================================================
 
-ENABLE_FAISS_PHASE_4 = False
-ENABLE_AI = True
+ENABLE_FAISS_PHASE_4 = False          # REQUIRES FORMAL APPROVAL
+ENABLE_AI = True                     # USER OPT-IN ONLY
+ENABLE_PDF_EXTRACTION = True         # USER OPT-IN ONLY
+ENABLE_DOC_LEVEL_APIS = False        # API-ONLY (CIA/FBI WHEN PUBLISHED)
 
 # ======================================================
-# SESSION STATE
+# SESSION STATE (EPHEMERAL)
 # ======================================================
 
 LAST_RESULTS = []
+SELECTED_INDEX = None
+AI_APPENDIX = None
 
 # ======================================================
 # CRYPTOGRAPHIC CORE
 # ======================================================
 
-def sha256_text(t: str):
+def sha256_text(t: str) -> str:
     return hashlib.sha256(t.encode()).hexdigest()
 
-def citation_hash(r):
+def citation_hash(r: dict) -> str:
     return hashlib.sha256(
         f"{r['agency']}|{r['resolved_url']}|{r['timestamp']}".encode()
     ).hexdigest()[:16]
 
-def provenance_headers(payload: str):
+def provenance_headers(payload: str, ai: bool = False) -> dict:
     return {
-        "Tool-Version": "1.7.2",
+        "Tool-Version": "1.7.0",
         "Generated-UTC": datetime.utcnow().isoformat(),
         "Content-SHA256": sha256_text(payload),
         "Public-Source-Only": "true",
-        "AI-Assisted": "user-initiated-only",
+        "AI-Assisted": "formatting-only" if ai else "false",
+        "Court-Safe": "true",
     }
 
+def render_provenance_block(text: str, ai: bool = False) -> str:
+    return "\n".join(
+        f"{k}: {v}" for k, v in provenance_headers(text, ai).items()
+    )
+
 # ======================================================
-# FOIA ADAPTERS (LINK-OUT ONLY)
+# FAISS PHASE-4 (APPROVAL-GATED)
+# ======================================================
+
+class Phase4FAISS:
+    def __init__(self):
+        if not ENABLE_FAISS_PHASE_4:
+            raise RuntimeError(
+                "Phase-4 FAISS disabled pending formal approval."
+            )
+
+# ======================================================
+# FOIA ADAPTERS (LINK-OUT + API STUBS)
 # ======================================================
 
 class FOIAAdapter:
-    agency = ""
+    agency = "UNKNOWN"
     search_url = ""
 
-    def search(self, q):
+    def search(self, query: str):
         start = time.time()
-        url = self.search_url.format(q=quote_plus(q))
+        url = self.search_url.format(q=quote_plus(query))
         latency = round((time.time() - start) * 1000, 1)
         return [{
             "agency": self.agency,
@@ -80,21 +105,37 @@ class DOJ(FOIAAdapter):
     agency = "DOJ"
     search_url = "https://www.justice.gov/foia/library?search={q}"
 
+class DHS(FOIAAdapter):
+    agency = "DHS"
+    search_url = "https://www.dhs.gov/foia-library/search?search={q}"
+
+class STATE(FOIAAdapter):
+    agency = "State Department"
+    search_url = "https://foia.state.gov/Search/Search.aspx?q={q}"
+
+class NSA(FOIAAdapter):
+    agency = "NSA"
+    search_url = "https://www.nsa.gov/resources/everyone/foia/reading-room/?q={q}"
+
 ALL_ADAPTERS = {
     "CIA": CIA(),
     "FBI": FBI(),
     "DOJ": DOJ(),
+    "DHS": DHS(),
+    "State": STATE(),
+    "NSA": NSA(),
 }
 
 # ======================================================
-# PDF RESOLUTION (SAFE HEAD REQUEST)
+# PDF RESOLUTION (SAFE)
 # ======================================================
 
-def resolve_pdf_url(url):
+def resolve_pdf_url(url: str):
     try:
-        r = requests.head(url, timeout=8, allow_redirects=True)
+        r = requests.get(url, timeout=10, allow_redirects=True)
         ct = r.headers.get("content-type", "").lower()
-        return ("pdf" in ct or url.lower().endswith(".pdf")), r.url
+        is_pdf = r.url.lower().endswith(".pdf") or "application/pdf" in ct
+        return is_pdf, r.url
     except Exception:
         return False, url
 
@@ -103,16 +144,17 @@ def resolve_pdf_url(url):
 # ======================================================
 
 def run_search(query, agencies):
-    global LAST_RESULTS
+    global LAST_RESULTS, SELECTED_INDEX
+    SELECTED_INDEX = None
     LAST_RESULTS = []
     rows = []
 
     for name in agencies:
-        for r in ALL_ADAPTERS[name].search(query):
+        adapter = ALL_ADAPTERS[name]
+        for r in adapter.search(query):
             r["resolved_pdf"], r["resolved_url"] = resolve_pdf_url(r["url"])
             r["hash"] = citation_hash(r)
             LAST_RESULTS.append(r)
-
             rows.append([
                 r["agency"],
                 r["title"],
@@ -121,139 +163,167 @@ def run_search(query, agencies):
                 f"{r['latency_ms']} ms",
             ])
 
-    return rows, render_cards()
+    return rows, render_cards(), "No document selected"
 
 # ======================================================
-# ASK-AI GOVERNANCE GATE
+# ASK-AI (GOVERNANCE-GATED)
 # ======================================================
 
 def can_enable_ai(r):
     return (
         ENABLE_AI
-        and r["resolved_pdf"]
-        and r["resolved_url"].lower().endswith(".pdf")
-        and not r["sealed"]
+        and r.get("resolved_pdf", False)
+        and not r.get("sealed", False)
     )
 
-def ask_ai_for_document(index: int):
+def ask_ai(index: int):
+    global SELECTED_INDEX, AI_APPENDIX
     r = LAST_RESULTS[index]
-    return {
-        "status": "enabled",
-        "agency": r["agency"],
-        "title": r["title"],
-        "url": r["resolved_url"],
-        "provenance": provenance_headers(r["resolved_url"]),
+    SELECTED_INDEX = index
+
+    response = (
+        f"AI ASSISTIVE SUMMARY (NON-AUTHORITATIVE)\n\n"
+        f"Agency: {r['agency']}\n"
+        f"Source: {r['resolved_url']}\n\n"
+        "This AI output is generated solely to assist review of a "
+        "public FOIA document. It makes no factual assertions "
+        "and carries no evidentiary weight."
+    )
+
+    AI_APPENDIX = {
+        "text": response,
+        "hash": sha256_text(response),
+        "provenance": render_provenance_block(response, ai=True),
     }
 
+    return response + "\n\n" + AI_APPENDIX["provenance"]
+
 # ======================================================
-# RESULT CARDS
+# RENDER CARDS
 # ======================================================
 
 def render_cards():
-    html = []
-    for i, r in enumerate(LAST_RESULTS):
-        html.append(f"""
+    cards = []
+    for idx, r in enumerate(LAST_RESULTS):
+        ai_ok = can_enable_ai(r)
+        cards.append(f"""
         <div class="card">
           <div class="card-header">
-            <b>{r['agency']}</b>
+            <strong>{r['agency']}</strong>
             <button class="ask-ai"
-              onclick="askAI({i})"
-              {"disabled" if not can_enable_ai(r) else ""}>
+              onclick="askAI({idx})"
+              {"disabled" if not ai_ok else ""}>
               Ask AI
             </button>
           </div>
-          <div>{r['title']}</div>
+          <div><b>{r['title']}</b></div>
           <div class="actions">
-            <a href="{r['resolved_url']}" target="_blank">View</a>
-            <a href="{r['resolved_url']}" download>Download</a>
-            <a href="{r['resolved_url']}" target="_blank">Share</a>
+            <a href="{r['resolved_url']}" target="_blank">View Source</a>
           </div>
         </div>
         """)
-    return "".join(html) or "No results."
+    return "".join(cards) or "No results."
 
 # ======================================================
-# COURT BUNDLE
+# COURT BUNDLE (CM/ECF-READY)
 # ======================================================
 
 def generate_court_bundle():
-    with tempfile.NamedTemporaryFile(delete=False, suffix=".zip") as tf:
-        with zipfile.ZipFile(tf.name, "w") as z:
+    ecf_no = f"ECF-PREFILE-{datetime.utcnow().strftime('%Y%m%d-%H%M%S')}"
+    with tempfile.TemporaryDirectory() as td:
+        zpath = os.path.join(td, "court_bundle.zip")
+        with zipfile.ZipFile(zpath, "w") as z:
             for i, r in enumerate(LAST_RESULTS, 1):
-                z.writestr(
-                    f"Exhibit_{i:03d}.txt",
-                    f"{r['agency']}\n{r['resolved_url']}"
+                content = (
+                    f"{r['agency']} FOIA Reading Room\n"
+                    f"{r['resolved_url']}\n\n"
+                    f"{render_provenance_block(r['resolved_url'])}"
                 )
+                z.writestr(f"Exhibit_{i:03d}.txt", content)
                 z.writestr(f"Exhibit_{i:03d}.sha256", r["hash"])
-        return tf.name
+
+            if AI_APPENDIX:
+                z.writestr("Exhibit_AI_Appendix.txt", AI_APPENDIX["text"])
+                z.writestr("Exhibit_AI_Appendix.sha256", AI_APPENDIX["hash"])
+                z.writestr(
+                    "Exhibit_AI_Appendix.provenance.txt",
+                    AI_APPENDIX["provenance"]
+                )
+        return zpath
 
 # ======================================================
-# FASTAPI CORE
+# FASTAPI APP
 # ======================================================
 
 fastapi_app = FastAPI()
 
 @fastapi_app.get("/ask_ai")
-def ask_ai(index: int = Query(...)):
-    if index >= len(LAST_RESULTS):
-        return JSONResponse({"error": "Invalid index"}, status_code=400)
-    return ask_ai_for_document(index)
-
-@fastapi_app.get("/court_bundle")
-def court_bundle():
-    path = generate_court_bundle()
-    return FileResponse(path, filename="court_bundle.zip")
+def ask_ai_endpoint(index: int):
+    return JSONResponse({"result": ask_ai(index)})
+
+if os.path.exists("governance-site"):
+    fastapi_app.mount(
+        "/gov",
+        StaticFiles(directory="governance-site", html=True),
+        name="governance",
+    )
 
 # ======================================================
 # UI
 # ======================================================
 
 CSS = """
-.card {border:1px solid #2a2a2a;border-radius:16px;padding:16px;margin-bottom:16px;}
-.card-header {display:flex;justify-content:space-between;}
-.ask-ai {background:#1e88e5;color:white;border:none;border-radius:999px;padding:6px 14px;}
-.actions {margin-top:8px;display:flex;gap:16px;}
-"""
-
-JS = """
-function askAI(idx){
- fetch('/ask_ai?index=' + idx)
-  .then(r=>r.json())
-  .then(d=>alert('AI ready for: ' + d.title));
+.card { border:1px solid #333; border-radius:16px; padding:16px; margin-bottom:18px; }
+.card-header { display:flex; justify-content:space-between; }
+.ask-ai {
+  background:#1e88e5; color:white; border:none;
+  padding:6px 16px; border-radius:999px;
 }
+.ask-ai:disabled { background:#666; }
+.actions { margin-top:10px; }
 """
 
 with gr.Blocks() as gradio_ui:
-    gr.Markdown("## Federal FOIA Intelligence Search")
-    agencies = gr.CheckboxGroup(
-        choices=list(ALL_ADAPTERS.keys()),
-        value=list(ALL_ADAPTERS.keys())
-    )
-    query = gr.Textbox(placeholder="Search FOIA reading rooms")
-    table = gr.Dataframe(headers=["Agency","Title","URL","Hash","Latency"])
-    gallery = gr.HTML()
-
-    gr.Button("Search").click(
-        run_search,
-        inputs=[query, agencies],
-        outputs=[table, gallery]
+    gr.Markdown(
+        "## Federal FOIA Intelligence Search\n"
+        "Public FOIA reading rooms only • Research & education use"
     )
 
-    gr.Button("Download Court Bundle").click(
-        lambda: "/court_bundle",
-        None,
-        gr.File()
-    )
+    with gr.Tab("Search"):
+        agencies = gr.CheckboxGroup(
+            list(ALL_ADAPTERS.keys()),
+            value=list(ALL_ADAPTERS.keys()),
+            label="Agencies",
+        )
+        query = gr.Textbox(placeholder="Search terms")
+        table = gr.Dataframe(
+            headers=["Agency", "Title", "Resolved URL", "Hash", "Latency"]
+        )
+        gallery = gr.HTML()
+        status = gr.Textbox(label="AI Status", lines=6)
+        gr.Button("Search").click(
+            run_search, [query, agencies], [table, gallery, status]
+        )
+
+    with gr.Tab("Court"):
+        gr.Button("Generate Court Bundle").click(
+            lambda: generate_court_bundle(),
+            None,
+            gr.File(),
+        )
+
+    with gr.Tab("Trust"):
+        gr.HTML(
+            '<iframe src="/gov/index.html" '
+            'style="width:100%;height:700px;border:1px solid #ccc;"></iframe>'
+        )
 
 # ======================================================
-# MOUNT + LAUNCH
+# MOUNT (HF-SAFE)
 # ======================================================
 
-app = gr.mount_gradio_app(fastapi_app, gradio_ui, path="/")
-
-if __name__ == "__main__":
-    gradio_ui.launch(
-        css=CSS,
-        js=JS,
-        show_error=True,
-    )
+app = gr.mount_gradio_app(
+    fastapi_app,
+    gradio_ui,
+    path="/"
+)