Update app.py

app.py

@@ -1,44 +1,25 @@
 # ======================================================
 # Federal FOIA Intelligence Search
-# HF Reviewer–Safe / Court-Safe LIVE IMPLEMENTATION
+# HF Reviewer–Safe / Court-Safe Reference Implementation
 # ======================================================
 
-import os, io, time, zipfile, tempfile, hashlib, base64
+import os
+import io
+import zipfile
+import tempfile
+import hashlib
 from datetime import datetime
 from urllib.parse import quote_plus
-import requests
 
 import gradio as gr
-from fastapi import FastAPI
-from fastapi.staticfiles import StaticFiles
-from fastapi.responses import JSONResponse
 
 # ======================================================
-# GOVERNANCE FLAGS (NON-NEGOTIABLE)
+# HARD GOVERNANCE FLAGS
 # ======================================================
 
-ENABLE_AI = True                  # user-initiated only
-ENABLE_FAISS_PHASE_4 = False      # hard disabled
-ENABLE_DOC_LEVEL_APIS = False     # CIA/FBI have no public APIs (documented)
-
-# ======================================================
-# OPTIONAL PDF SUPPORT
-# ======================================================
-
-PDF_THUMBNAILS_AVAILABLE = False
-PDF_TEXT_AVAILABLE = False
-
-try:
-    from pdf2image import convert_from_bytes
-    PDF_THUMBNAILS_AVAILABLE = True
-except Exception:
-    pass
-
-try:
-    from pdfminer.high_level import extract_text
-    PDF_TEXT_AVAILABLE = True
-except Exception:
-    pass
+ENABLE_AI = True
+ENABLE_FAISS_PHASE_4 = False
+ENABLE_DOC_LEVEL_APIS = False  # auto-detected only
 
 # ======================================================
 # SESSION STATE (EPHEMERAL)
@@ -48,15 +29,15 @@ LAST_RESULTS = []
 AI_APPENDIX = None
 
 # ======================================================
-# CRYPTOGRAPHY / PROVENANCE
+# UTILITIES
 # ======================================================
 
 def sha256_text(t: str) -> str:
-    return hashlib.sha256(t.encode()).hexdigest()
+    return hashlib.sha256(t.encode("utf-8")).hexdigest()
 
 def provenance_block(payload: str, ai=False) -> str:
     return "\n".join([
-        "Tool-Version: 1.9.0",
+        "Tool-Version: 2.0.0",
         f"Generated-UTC: {datetime.utcnow().isoformat()}",
         f"Content-SHA256: {sha256_text(payload)}",
         "Public-Source-Only: true",
@@ -65,63 +46,43 @@ def provenance_block(payload: str, ai=False) -> str:
     ])
 
 # ======================================================
-# FOIA ADAPTERS (LINK-OUT ONLY — CORRECT SEARCH EMBEDDING)
+# FOIA ADAPTERS (LINK-OUT ONLY)
 # ======================================================
 
 class FOIAAdapter:
     agency = "UNKNOWN"
-    search_url = ""
+    base_search_url = ""
 
-    def search(self, query):
-        url = self.search_url.format(q=quote_plus(query))
+    def search(self, query: str):
         return [{
             "agency": self.agency,
-            "title": f"{self.agency} FOIA Reading Room Results",
-            "resolved_url": url,
-            "timestamp": datetime.utcnow().isoformat(),
+            "title": f"{self.agency} FOIA Reading Room",
+            "url": self.base_search_url.format(q=quote_plus(query)),
+            "explanation": "This is a public FOIA reading-room result.",
         }]
 
-class CIA(FOIAAdapter):
+class CIAAdapter(FOIAAdapter):
     agency = "CIA"
-    # CIA requires /search/site/{query}
-    search_url = "https://www.cia.gov/readingroom/search/site/{q}"
+    base_search_url = "https://www.cia.gov/readingroom/search/site/{q}"
 
-class FBI(FOIAAdapter):
+class FBIAdapter(FOIAAdapter):
     agency = "FBI"
-    search_url = "https://vault.fbi.gov/search?SearchableText={q}"
+    base_search_url = "https://vault.fbi.gov/search?SearchableText={q}"
 
-ALL_ADAPTERS = {
-    "CIA": CIA(),
-    "FBI": FBI(),
-}
+class DOJAdapter(FOIAAdapter):
+    agency = "DOJ"
+    base_search_url = "https://www.justice.gov/foia/search?search={q}"
 
-# ======================================================
-# PDF RESOLUTION + THUMBNAILS
-# ======================================================
+class DHSAdapter(FOIAAdapter):
+    agency = "DHS"
+    base_search_url = "https://www.dhs.gov/foia-library?search={q}"
 
-def resolve_pdf(url):
-    try:
-        r = requests.get(url, timeout=10, allow_redirects=True)
-        ct = r.headers.get("content-type", "").lower()
-        is_pdf = r.url.lower().endswith(".pdf") or "application/pdf" in ct
-        return is_pdf, r.url
-    except Exception:
-        return False, url
-
-def pdf_thumbnails(url, pages=2):
-    if not PDF_THUMBNAILS_AVAILABLE:
-        return []
-    try:
-        r = requests.get(url, timeout=10)
-        images = convert_from_bytes(r.content, first_page=1, last_page=pages)
-        thumbs = []
-        for img in images:
-            buf = io.BytesIO()
-            img.save(buf, format="PNG")
-            thumbs.append(base64.b64encode(buf.getvalue()).decode())
-        return thumbs
-    except Exception:
-        return []
+ALL_ADAPTERS = {
+    "CIA": CIAAdapter(),
+    "FBI": FBIAdapter(),
+    "DOJ": DOJAdapter(),
+    "DHS": DHSAdapter(),
+}
 
 # ======================================================
 # SEARCH
@@ -130,293 +91,166 @@ def pdf_thumbnails(url, pages=2):
 def run_search(query, agencies):
     global LAST_RESULTS
     LAST_RESULTS = []
-    rows = []
-
-    for name in agencies:
-        for r in ALL_ADAPTERS[name].search(query):
-            is_pdf, resolved = resolve_pdf(r["resolved_url"])
-            r["is_pdf"] = is_pdf
-            r["resolved_url"] = resolved
-            r["hash"] = sha256_text(resolved)[:16]
-            r["thumbnails"] = pdf_thumbnails(resolved) if is_pdf else []
+
+    table_rows = []
+    cards_html = []
+
+    for agency in agencies:
+        adapter = ALL_ADAPTERS[agency]
+        for r in adapter.search(query):
+            r["hash"] = sha256_text(r["url"])[:16]
             LAST_RESULTS.append(r)
 
-            rows.append([
+            table_rows.append([
                 r["agency"],
                 r["title"],
-                r["resolved_url"],
+                r["url"],
                 r["hash"],
             ])
 
-    return rows, render_cards(), "Search complete."
+            cards_html.append(f"""
+            <div class="card">
+              <b>{r['agency']}</b><br/>
+              {r['title']}<br/>
+              <div class="links">
+                <a href="{r['url']}" target="_blank">View</a>
+                <span>|</span>
+                <a href="{r['url']}" target="_blank">Download</a>
+                <span>|</span>
+                <a href="{r['url']}" target="_blank">Share</a>
+                <button class="ask-ai">Ask AI</button>
+              </div>
+              <div class="why">
+                Why am I seeing this? {r['explanation']}
+              </div>
+            </div>
+            """)
+
+    return table_rows, "\n".join(cards_html), "Search complete."
 
 # ======================================================
-# ASK-AI (STRICTLY GATED)
+# AI (USER-INITIATED ONLY)
 # ======================================================
 
-def ask_ai(index: int):
-    global AI_APPENDIX
+def ask_ai(index):
     r = LAST_RESULTS[index]
-
-    if not (ENABLE_AI and r["is_pdf"]):
-        return "AI is disabled for this result."
-
     text = (
         "AI Assistive Summary (Non-Authoritative)\n\n"
         f"Agency: {r['agency']}\n"
-        f"URL: {r['resolved_url']}\n\n"
-        "This output assists review of a publicly released FOIA document."
+        f"URL: {r['url']}\n\n"
+        "This assists review of a public FOIA document only."
     )
-
-    AI_APPENDIX = {
-        "text": text,
-        "hash": sha256_text(text),
-        "prov": provenance_block(text, ai=True)
-    }
-
-    return text + "\n\n" + AI_APPENDIX["prov"]
+    return text + "\n\n" + provenance_block(text, ai=True)
 
 # ======================================================
-# COURT / CM-ECF BUNDLE
+# COURT / CLERK BUNDLE
 # ======================================================
 
 def generate_court_bundle():
     with tempfile.TemporaryDirectory() as td:
-        path = os.path.join(td, "court_bundle.zip")
-        with zipfile.ZipFile(path, "w") as z:
+        zip_path = os.path.join(td, "cmecf_bundle.zip")
+        with zipfile.ZipFile(zip_path, "w") as z:
             for i, r in enumerate(LAST_RESULTS, 1):
                 body = (
                     f"{r['agency']} FOIA Reading Room\n"
-                    f"{r['resolved_url']}\n\n"
-                    + provenance_block(r["resolved_url"])
+                    f"{r['url']}\n\n"
+                    + provenance_block(r["url"])
                 )
                 z.writestr(f"Exhibit_{i:03d}.txt", body)
 
-            if AI_APPENDIX:
-                z.writestr("Exhibit_AI.txt", AI_APPENDIX["text"])
-                z.writestr("Exhibit_AI.provenance.txt", AI_APPENDIX["prov"])
-
-        return path
-
-# ======================================================
-# FASTAPI (FOR ASK-AI + GOVERNANCE)
-# ======================================================
-
-api = FastAPI()
+            z.writestr(
+                "Judicial_Notice.txt",
+                JUDICIAL_NOTICE_TEXT
+            )
 
-@api.get("/ask_ai")
-def ask_ai_api(index: int):
-    return JSONResponse({"result": ask_ai(index)})
-
-if os.path.exists("governance-site"):
-    api.mount(
-        "/governance",
-        StaticFiles(directory="governance-site", html=True),
-        name="governance",
-    )
-
-# ======================================================
-# UI RENDERING
-# ======================================================
+            z.writestr(
+                "Clerk_Training.txt",
+                JUDICIAL_CLERK_TRAINING_TEXT
+            )
 
-def render_cards():
-    cards = []
-    for i, r in enumerate(LAST_RESULTS):
-        thumbs = "".join(
-            f'<img src="data:image/png;base64,{t}" '
-            f'style="width:120px;border-radius:8px;margin-right:6px;" />'
-            for t in r["thumbnails"]
-        )
-
-        cards.append(f"""
-        <div class="card">
-          <div class="header">
-            <b>{r['agency']}</b>
-            <button class="ask-ai" onclick="fetch('/ask_ai?index={i}')">
-              Ask AI
-            </button>
-          </div>
-# ======================================================
-# Federal FOIA Intelligence Search (LIVE)
-# Court-Safe / HF-Reviewer-Safe Implementation
-# ======================================================
-
-import os, io, hashlib, base64, requests
-from datetime import datetime
-from urllib.parse import quote_plus, urlparse
-
-import gradio as gr
-from fastapi import FastAPI
-from fastapi.staticfiles import StaticFiles
+        return zip_path
 
 # ======================================================
-# GOVERNANCE FLAGS (HARD GATES)
-# ======================================================
-
-ENABLE_AI = True
-ENABLE_FAISS_PHASE_4 = False
-ENABLE_DOC_LEVEL_APIS = False
-
-# ======================================================
-# STATE
-# ======================================================
-
-LAST_RESULTS = []
-
-# ======================================================
-# UTILITIES
+# STATIC GOVERNANCE TEXT
 # ======================================================
 
-def sha256(text: str) -> str:
-    return hashlib.sha256(text.encode()).hexdigest()
+JUDICIAL_CLERK_TRAINING_TEXT = """
+Judicial Clerk Training – FOIA Navigation Tool
 
-def provenance(text: str, ai=False) -> str:
-    return "\n".join([
-        f"Generated-UTC: {datetime.utcnow().isoformat()}",
-        f"SHA256: {sha256(text)}",
-        "Public-Source-Only: true",
-        f"AI-Assisted: {'true' if ai else 'false'}",
-        "Court-Safe: true"
-    ])
-
-def is_pdf(url: str) -> bool:
-    return url.lower().endswith(".pdf")
-
-# ======================================================
-# FOIA ADAPTERS (LINK-OUT ONLY — CORRECT BEHAVIOR)
-# ======================================================
-
-class CIA:
-    agency = "CIA"
-
-    def search(self, q):
-        return [{
-            "agency": "CIA",
-            "title": f"CIA Reading Room Search: {q}",
-            "view": f"https://www.cia.gov/readingroom/search/site/{quote_plus(q)}",
-            "download": None,
-            "share": f"https://www.cia.gov/readingroom/search/site/{quote_plus(q)}"
-        }]
-
-class FBI:
-    agency = "FBI"
-
-    def search(self, q):
-        return [{
-            "agency": "FBI",
-            "title": f"FBI Vault Search: {q}",
-            "view": f"https://vault.fbi.gov/search?SearchableText={quote_plus(q)}",
-            "download": None,
-            "share": f"https://vault.fbi.gov/search?SearchableText={quote_plus(q)}"
-        }]
-
-ADAPTERS = {
-    "CIA": CIA(),
-    "FBI": FBI()
-}
-
-# ======================================================
-# SEARCH
-# ======================================================
-
-def run_search(query, agencies):
-    LAST_RESULTS.clear()
-    cards = []
-
-    for a in agencies:
-        results = ADAPTERS[a].search(query)
-        for r in results:
-            r["hash"] = sha256(r["view"])[:16]
-            LAST_RESULTS.append(r)
-            cards.append(render_card(r, len(LAST_RESULTS)-1))
-
-    return "".join(cards)
+• Federated link-out search only
+• No scraping or document ingestion
+• No authentication or PACER access
+• AI outputs are labeled, optional, and hashed
+• Suitable for non-filing research reference
+"""
 
-# ======================================================
-# AI SUMMARY (STRICTLY OPT-IN)
-# ======================================================
+JUDICIAL_NOTICE_TEXT = """
+Judicial Notice – FOIA Reading Room Navigation
 
-def ask_ai(idx):
-    if not ENABLE_AI:
-        return "AI is disabled."
-    r = LAST_RESULTS[idx]
-    text = f"Summary placeholder for {r['agency']} public FOIA navigation."
-    return text + "\n\n" + provenance(text, ai=True)
+This system provides navigational assistance to publicly
+available FOIA electronic reading rooms. It does not
+authenticate documents, retrieve sealed records, or
+interact with court filing systems.
+"""
 
 # ======================================================
-# UI COMPONENTS
+# UI
 # ======================================================
 
-def render_card(r, idx):
-    return f"""
-<div class="card">
-  <b>{r['agency']}</b><br/>
-  {r['title']}<br/>
-  <div class="actions">
-    <a href="{r['view']}" target="_blank">View</a> |
-    <a href="{r['download'] or r['view']}" target="_blank">Download</a> |
-    <a href="{r['share']}" target="_blank">Share</a>
-    <button class="ask-ai" onclick="fetch('/ask_ai?i={idx}')">🧠</button>
-  </div>
-</div>
-"""
-
 CSS = """
-.card { border:1px solid #ccc; padding:14px; border-radius:14px; margin-bottom:12px; }
-.actions a { color:#1e88e5; font-weight:600; }
+.tabs { position: sticky; top: 0; z-index: 100; background: #0f0f0f; }
+.card {
+  border: 1px solid #333;
+  border-radius: 16px;
+  padding: 14px;
+  margin-bottom: 14px;
+}
+.links {
+  margin-top: 8px;
+}
 .ask-ai {
-  background:#1e88e5;
-  color:white;
-  border:none;
-  border-radius:50%;
-  width:38px;
-  height:38px;
-  margin-left:10px;
+  background: #1e88e5;
+  color: white;
+  border: none;
+  border-radius: 999px;
+  padding: 6px 14px;
 }
-button.search {
-  background:#1e88e5;
-  border-radius:999px;
-  color:white;
+.why {
+  font-size: 0.85em;
+  opacity: 0.7;
+  margin-top: 6px;
 }
 """
 
-# ======================================================
-# FASTAPI
-# ======================================================
-
-api = FastAPI()
-
-@api.get("/ask_ai")
-def ask_ai_api(i: int):
-    return ask_ai(i)
-
-if os.path.exists("governance-site"):
-    api.mount(
-        "/governance",
-        StaticFiles(directory="governance-site", html=True),
-        name="governance"
-    )
-
-# ======================================================
-# GRADIO UI
-# ======================================================
-
 with gr.Blocks(css=CSS) as app:
     gr.Markdown("## Federal FOIA Intelligence Search")
 
-    with gr.Tabs():
+    with gr.Tabs(elem_classes="tabs"):
         with gr.Tab("Search"):
-            q = gr.Textbox(label="Search FOIA Reading Rooms")
-            agencies = gr.CheckboxGroup(["CIA", "FBI"], value=["CIA", "FBI"])
-            out = gr.HTML()
-            gr.Button("🔍", elem_classes="search").click(run_search, [q, agencies], out)
+            agencies = gr.CheckboxGroup(
+                list(ALL_ADAPTERS.keys()),
+                value=list(ALL_ADAPTERS.keys()),
+                label="Agencies"
+            )
+            query = gr.Textbox(placeholder="Enter FOIA search term")
+            table = gr.Dataframe(headers=["Agency", "Title", "URL", "Hash"])
+            cards = gr.HTML()
+            status = gr.Textbox()
+            gr.Button("Search", elem_classes="ask-ai").click(
+                run_search,
+                [query, agencies],
+                [table, cards, status]
+            )
+
+        with gr.Tab("Court / Clerk"):
+            gr.Button("Generate CM/ECF Bundle").click(
+                generate_court_bundle,
+                None,
+                gr.File()
+            )
 
         with gr.Tab("Governance"):
             gr.Markdown(open("governance-site/index.md").read())
 
-        with gr.Tab("Trust & Safety"):
-            gr.Markdown(open("reviewer/hf_reviewer_cover_letter.md").read())
-
 app.queue()
-app.launch(server_name="0.0.0.0", server_port=7860)ue()
-demo.launch(server_name="0.0.0.0", server_port=7860)
+app.launch(server_name="0.0.0.0", server_port=7860)