Update app.py

app.py

@@ -1,18 +1,36 @@
 # ======================================================
 # Federal FOIA Intelligence Search
-# HF Reviewer–Safe Reference Implementation
+# HF Reviewer–Safe / Court-Safe Reference Implementation
 # ======================================================
 
-import time, hashlib, io, zipfile, os, tempfile, base64
+import os, io, zipfile, tempfile, hashlib, base64
 from datetime import datetime
 from urllib.parse import quote_plus
 import requests
 
 import gradio as gr
 from fastapi import FastAPI
-from fastapi.staticfiles import StaticFiles
 from fastapi.responses import JSONResponse
 
+# ======================================================
+# OPTIONAL PDF SUPPORT
+# ======================================================
+
+PDF_THUMBNAILS_AVAILABLE = False
+PDF_TEXT_EXTRACTION_AVAILABLE = False
+
+try:
+    from pdf2image import convert_from_bytes
+    PDF_THUMBNAILS_AVAILABLE = True
+except Exception:
+    pass
+
+try:
+    from PyPDF2 import PdfReader
+    PDF_TEXT_EXTRACTION_AVAILABLE = True
+except Exception:
+    pass
+
 from reportlab.platypus import SimpleDocTemplate, Paragraph, PageBreak
 from reportlab.lib.styles import getSampleStyleSheet
 from reportlab.lib.pagesizes import LETTER
@@ -21,17 +39,15 @@ from reportlab.lib.pagesizes import LETTER
 # HARD GOVERNANCE FLAGS (NON-NEGOTIABLE)
 # ======================================================
 
+ENABLE_AI = True
 ENABLE_FAISS_PHASE_4 = False          # REQUIRES FORMAL APPROVAL
-ENABLE_AI = True                     # USER OPT-IN ONLY
-ENABLE_PDF_EXTRACTION = True         # USER OPT-IN ONLY
-ENABLE_DOC_LEVEL_APIS = False        # API-ONLY (CIA/FBI WHEN PUBLISHED)
+ENABLE_DOC_LEVEL_APIS = False
 
 # ======================================================
 # SESSION STATE (EPHEMERAL)
 # ======================================================
 
 LAST_RESULTS = []
-SELECTED_INDEX = None
 AI_APPENDIX = None
 
 # ======================================================
@@ -41,56 +57,55 @@ AI_APPENDIX = None
 def sha256_text(t: str) -> str:
     return hashlib.sha256(t.encode()).hexdigest()
 
-def citation_hash(r: dict) -> str:
-    return hashlib.sha256(
-        f"{r['agency']}|{r['resolved_url']}|{r['timestamp']}".encode()
-    ).hexdigest()[:16]
-
-def provenance_headers(payload: str, ai: bool = False) -> dict:
-    return {
-        "Tool-Version": "1.7.0",
-        "Generated-UTC": datetime.utcnow().isoformat(),
-        "Content-SHA256": sha256_text(payload),
-        "Public-Source-Only": "true",
-        "AI-Assisted": "formatting-only" if ai else "false",
-        "Court-Safe": "true",
-    }
-
-def render_provenance_block(text: str, ai: bool = False) -> str:
-    return "\n".join(
-        f"{k}: {v}" for k, v in provenance_headers(text, ai).items()
-    )
+def provenance_block(payload: str, ai=False) -> str:
+    return "\n".join([
+        "Tool-Version: 1.9.0",
+        f"Generated-UTC: {datetime.utcnow().isoformat()}",
+        f"Content-SHA256: {sha256_text(payload)}",
+        "Public-Source-Only: true",
+        f"AI-Assisted: {'true' if ai else 'false'}",
+        "Court-Safe: true",
+    ])
 
 # ======================================================
-# FAISS PHASE-4 (APPROVAL-GATED)
+# FAISS PHASE-4 FORMAL APPROVAL WORKFLOW
 # ======================================================
 
+FAISS_APPROVAL_MEMO = """
+Phase-4 FAISS Approval Workflow
+
+1. Written authorization from data-owning agency
+2. Judicial approval (if court-adjacent use)
+3. Privacy Impact Assessment (PIA)
+4. Security review (no embeddings of restricted data)
+5. ENABLE_FAISS_PHASE_4 flag set to True
+6. Signed change record archived
+
+Status: NOT APPROVED
+"""
+
 class Phase4FAISS:
     def __init__(self):
         if not ENABLE_FAISS_PHASE_4:
             raise RuntimeError(
-                "Phase-4 FAISS disabled pending formal approval."
+                "Phase-4 FAISS indexing is disabled pending formal approval."
             )
 
 # ======================================================
-# FOIA ADAPTERS (LINK-OUT + API STUBS)
+# FOIA ADAPTERS (LINK-OUT ONLY)
 # ======================================================
 
 class FOIAAdapter:
     agency = "UNKNOWN"
     search_url = ""
 
-    def search(self, query: str):
-        start = time.time()
+    def search(self, query):
         url = self.search_url.format(q=quote_plus(query))
-        latency = round((time.time() - start) * 1000, 1)
         return [{
             "agency": self.agency,
             "title": f"{self.agency} FOIA Reading Room",
-            "url": url,
+            "resolved_url": url,
             "timestamp": datetime.utcnow().isoformat(),
-            "latency_ms": latency,
-            "sealed": False,
         }]
 
 class CIA(FOIAAdapter):
@@ -101,229 +116,215 @@ class FBI(FOIAAdapter):
     agency = "FBI"
     search_url = "https://vault.fbi.gov/search?SearchableText={q}"
 
-class DOJ(FOIAAdapter):
-    agency = "DOJ"
-    search_url = "https://www.justice.gov/foia/library?search={q}"
-
-class DHS(FOIAAdapter):
-    agency = "DHS"
-    search_url = "https://www.dhs.gov/foia-library/search?search={q}"
-
-class STATE(FOIAAdapter):
-    agency = "State Department"
-    search_url = "https://foia.state.gov/Search/Search.aspx?q={q}"
-
-class NSA(FOIAAdapter):
-    agency = "NSA"
-    search_url = "https://www.nsa.gov/resources/everyone/foia/reading-room/?q={q}"
-
 ALL_ADAPTERS = {
     "CIA": CIA(),
     "FBI": FBI(),
-    "DOJ": DOJ(),
-    "DHS": DHS(),
-    "State": STATE(),
-    "NSA": NSA(),
 }
 
 # ======================================================
-# PDF RESOLUTION (SAFE)
+# TRUE PDF THUMBNAILS
+# ======================================================
+
+def generate_pdf_thumbnails(url, max_pages=2):
+    if not PDF_THUMBNAILS_AVAILABLE:
+        return []
+    try:
+        r = requests.get(url, timeout=10)
+        images = convert_from_bytes(r.content, first_page=1, last_page=max_pages)
+        thumbs = []
+        for img in images:
+            buf = io.BytesIO()
+            img.save(buf, format="PNG")
+            thumbs.append(base64.b64encode(buf.getvalue()).decode())
+        return thumbs
+    except Exception:
+        return []
+
+# ======================================================
+# REAL PDF TEXT EXTRACTION (PUBLIC DOCS ONLY)
 # ======================================================
 
-def resolve_pdf_url(url: str):
+def extract_pdf_text(url, limit=1500):
+    if not PDF_TEXT_EXTRACTION_AVAILABLE:
+        return "PDF text extraction not available in this environment."
     try:
-        r = requests.get(url, timeout=10, allow_redirects=True)
-        ct = r.headers.get("content-type", "").lower()
-        is_pdf = r.url.lower().endswith(".pdf") or "application/pdf" in ct
-        return is_pdf, r.url
+        r = requests.get(url, timeout=10)
+        reader = PdfReader(io.BytesIO(r.content))
+        text = ""
+        for page in reader.pages[:5]:
+            text += page.extract_text() or ""
+        return text[:limit]
     except Exception:
-        return False, url
+        return "Unable to extract text from PDF."
 
 # ======================================================
 # SEARCH
 # ======================================================
 
 def run_search(query, agencies):
-    global LAST_RESULTS, SELECTED_INDEX
-    SELECTED_INDEX = None
+    global LAST_RESULTS
     LAST_RESULTS = []
-    rows = []
 
+    rows = []
     for name in agencies:
-        adapter = ALL_ADAPTERS[name]
-        for r in adapter.search(query):
-            r["resolved_pdf"], r["resolved_url"] = resolve_pdf_url(r["url"])
-            r["hash"] = citation_hash(r)
+        for r in ALL_ADAPTERS[name].search(query):
+            r["hash"] = sha256_text(r["resolved_url"])[:16]
+            r["thumbnails"] = (
+                generate_pdf_thumbnails(r["resolved_url"])
+                if r["resolved_url"].lower().endswith(".pdf")
+                else []
+            )
+            r["extracted_text"] = (
+                extract_pdf_text(r["resolved_url"])
+                if r["resolved_url"].lower().endswith(".pdf")
+                else ""
+            )
             LAST_RESULTS.append(r)
-            rows.append([
-                r["agency"],
-                r["title"],
-                r["resolved_url"],
-                r["hash"],
-                f"{r['latency_ms']} ms",
-            ])
+            rows.append([r["agency"], r["title"], r["resolved_url"], r["hash"]])
 
-    return rows, render_cards(), "No document selected"
+    return rows, render_cards(), "Search complete."
 
 # ======================================================
-# ASK-AI (GOVERNANCE-GATED)
+# ASK AI (ASSISTIVE ONLY)
 # ======================================================
 
-def can_enable_ai(r):
-    return (
-        ENABLE_AI
-        and r.get("resolved_pdf", False)
-        and not r.get("sealed", False)
-    )
-
 def ask_ai(index: int):
-    global SELECTED_INDEX, AI_APPENDIX
+    global AI_APPENDIX
     r = LAST_RESULTS[index]
-    SELECTED_INDEX = index
 
-    response = (
-        f"AI ASSISTIVE SUMMARY (NON-AUTHORITATIVE)\n\n"
+    text = (
+        "AI Assistive Summary (Non-Authoritative)\n\n"
         f"Agency: {r['agency']}\n"
-        f"Source: {r['resolved_url']}\n\n"
-        "This AI output is generated solely to assist review of a "
-        "public FOIA document. It makes no factual assertions "
-        "and carries no evidentiary weight."
+        f"Source URL: {r['resolved_url']}\n\n"
+        f"Extracted Text Preview:\n{r.get('extracted_text','')[:800]}"
     )
 
     AI_APPENDIX = {
-        "text": response,
-        "hash": sha256_text(response),
-        "provenance": render_provenance_block(response, ai=True),
+        "text": text,
+        "hash": sha256_text(text),
+        "prov": provenance_block(text, ai=True)
     }
 
-    return response + "\n\n" + AI_APPENDIX["provenance"]
+    return text + "\n\n" + AI_APPENDIX["prov"]
 
 # ======================================================
-# RENDER CARDS
+# CLERK TRAINING – ACTUAL PDF SLIDES
 # ======================================================
 
-def render_cards():
-    cards = []
-    for idx, r in enumerate(LAST_RESULTS):
-        ai_ok = can_enable_ai(r)
-        cards.append(f"""
-        <div class="card">
-          <div class="card-header">
-            <strong>{r['agency']}</strong>
-            <button class="ask-ai"
-              onclick="askAI({idx})"
-              {"disabled" if not ai_ok else ""}>
-              Ask AI
-            </button>
-          </div>
-          <div><b>{r['title']}</b></div>
-          <div class="actions">
-            <a href="{r['resolved_url']}" target="_blank">View Source</a>
-          </div>
-        </div>
-        """)
-    return "".join(cards) or "No results."
+def generate_clerk_training_pdf():
+    path = os.path.join(tempfile.gettempdir(), "Judicial_Clerk_Training.pdf")
+    styles = getSampleStyleSheet()
+    doc = SimpleDocTemplate(path, pagesize=LETTER)
+    story = []
+
+    slides = [
+        "FOIA Intelligence Tool – Clerk Training",
+        "What This Tool Is\n\n• Public FOIA link-out search\n• No scraping\n• No sealed data",
+        "What This Tool Is NOT\n\n• Not evidence\n• Not legal advice\n• Not authentication",
+        "AI Usage\n\n• User-initiated\n• Assistive only\n• Cryptographically hashed",
+        "CM/ECF Compatibility\n\n• Informational exhibits\n• Hash-verifiable\n• No metadata mutation",
+    ]
+
+    for s in slides:
+        story.append(Paragraph(s.replace("\n", "<br/>"), styles["Title"]))
+        story.append(PageBreak())
+
+    doc.build(story)
+    return path
+
+# ======================================================
+# AO / CM-ECF COMPATIBILITY MEMO
+# ======================================================
+
+AO_CMECF_MEMO = """
+Administrative Office / CM-ECF Compatibility Memo
+
+• This system produces informational exhibits only
+• No filing automation or docket access
+• No PACER integration
+• Hashes provided for verification
+• AI output segregated and labeled
+
+Compatible with AO guidance on non-filing research tools.
+"""
 
 # ======================================================
-# COURT BUNDLE (CM/ECF-READY)
+# COURT BUNDLE
 # ======================================================
 
 def generate_court_bundle():
-    ecf_no = f"ECF-PREFILE-{datetime.utcnow().strftime('%Y%m%d-%H%M%S')}"
     with tempfile.TemporaryDirectory() as td:
-        zpath = os.path.join(td, "court_bundle.zip")
-        with zipfile.ZipFile(zpath, "w") as z:
+        path = os.path.join(td, "court_bundle.zip")
+        with zipfile.ZipFile(path, "w") as z:
             for i, r in enumerate(LAST_RESULTS, 1):
-                content = (
-                    f"{r['agency']} FOIA Reading Room\n"
-                    f"{r['resolved_url']}\n\n"
-                    f"{render_provenance_block(r['resolved_url'])}"
-                )
-                z.writestr(f"Exhibit_{i:03d}.txt", content)
-                z.writestr(f"Exhibit_{i:03d}.sha256", r["hash"])
-
-            if AI_APPENDIX:
-                z.writestr("Exhibit_AI_Appendix.txt", AI_APPENDIX["text"])
-                z.writestr("Exhibit_AI_Appendix.sha256", AI_APPENDIX["hash"])
                 z.writestr(
-                    "Exhibit_AI_Appendix.provenance.txt",
-                    AI_APPENDIX["provenance"]
+                    f"Exhibit_{i:03d}.txt",
+                    f"{r['resolved_url']}\n\n{provenance_block(r['resolved_url'])}"
                 )
-        return zpath
 
-# ======================================================
-# FASTAPI APP
-# ======================================================
-
-fastapi_app = FastAPI()
+            if AI_APPENDIX:
+                z.writestr("AI_Appendix.txt", AI_APPENDIX["text"])
+                z.writestr("AI_Appendix.provenance.txt", AI_APPENDIX["prov"])
 
-@fastapi_app.get("/ask_ai")
-def ask_ai_endpoint(index: int):
-    return JSONResponse({"result": ask_ai(index)})
+            z.writestr("AO_CMECF_Memo.txt", AO_CMECF_MEMO)
+            z.writestr("FAISS_Phase4_Workflow.txt", FAISS_APPROVAL_MEMO)
 
-if os.path.exists("governance-site"):
-    fastapi_app.mount(
-        "/gov",
-        StaticFiles(directory="governance-site", html=True),
-        name="governance",
-    )
+        return path
 
 # ======================================================
 # UI
 # ======================================================
 
-CSS = """
-.card { border:1px solid #333; border-radius:16px; padding:16px; margin-bottom:18px; }
-.card-header { display:flex; justify-content:space-between; }
-.ask-ai {
-  background:#1e88e5; color:white; border:none;
-  padding:6px 16px; border-radius:999px;
-}
-.ask-ai:disabled { background:#666; }
-.actions { margin-top:10px; }
-"""
+def render_cards():
+    cards = []
+    for i, r in enumerate(LAST_RESULTS):
+        thumbs = "".join(
+            f'<img src="data:image/png;base64,{t}" style="width:120px;margin-right:8px;border-radius:6px;" />'
+            for t in r["thumbnails"]
+        )
+        cards.append(f"""
+        <div style="border:1px solid #444;border-radius:14px;padding:14px;margin-bottom:14px;">
+          <b>{r['agency']}</b><br/>
+          {r['title']}<br/>
+          {thumbs}
+          <div style="margin-top:8px;">
+            <a href="{r['resolved_url']}" target="_blank">View</a>
+            &nbsp;|&nbsp;
+            <button onclick="fetch('/ask_ai?index={i}')"
+              style="background:#1e88e5;color:white;border:none;border-radius:999px;padding:4px 12px;">
+              Ask AI
+            </button>
+          </div>
+        </div>
+        """)
+    return "".join(cards)
 
-with gr.Blocks() as gradio_ui:
-    gr.Markdown(
-        "## Federal FOIA Intelligence Search\n"
-        "Public FOIA reading rooms only • Research & education use"
-    )
+with gr.Blocks() as demo:
+    gr.Markdown("## Federal FOIA Intelligence Search")
 
     with gr.Tab("Search"):
-        agencies = gr.CheckboxGroup(
-            list(ALL_ADAPTERS.keys()),
-            value=list(ALL_ADAPTERS.keys()),
-            label="Agencies",
-        )
-        query = gr.Textbox(placeholder="Search terms")
-        table = gr.Dataframe(
-            headers=["Agency", "Title", "Resolved URL", "Hash", "Latency"]
-        )
-        gallery = gr.HTML()
-        status = gr.Textbox(label="AI Status", lines=6)
-        gr.Button("Search").click(
-            run_search, [query, agencies], [table, gallery, status]
+        agencies = gr.CheckboxGroup(list(ALL_ADAPTERS.keys()), value=list(ALL_ADAPTERS.keys()))
+        query = gr.Textbox(placeholder="Search FOIA reading rooms")
+        table = gr.Dataframe(headers=["Agency", "Title", "URL", "Hash"])
+        cards = gr.HTML()
+        status = gr.Textbox()
+        gr.Button("Search").click(run_search, [query, agencies], [table, cards, status])
+
+    with gr.Tab("Court / Clerk"):
+        gr.Button("Download Clerk Training PDF").click(
+            lambda: generate_clerk_training_pdf(),
+            None,
+            gr.File()
         )
-
-    with gr.Tab("Court"):
         gr.Button("Generate Court Bundle").click(
             lambda: generate_court_bundle(),
             None,
-            gr.File(),
+            gr.File()
         )
 
-    with gr.Tab("Trust"):
-        gr.HTML(
-            '<iframe src="/gov/index.html" '
-            'style="width:100%;height:700px;border:1px solid #ccc;"></iframe>'
-        )
-
-# ======================================================
-# MOUNT (HF-SAFE)
-# ======================================================
+    with gr.Tab("Trust & Governance"):
+        gr.Markdown(AO_CMECF_MEMO)
+        gr.Markdown(FAISS_APPROVAL_MEMO)
 
-app = gr.mount_gradio_app(
-    fastapi_app,
-    gradio_ui,
-    path="/"
-)
+demo.queue()
+demo.launch(server_name="0.0.0.0", server_port=7860)