Update app.py

app.py

@@ -1,34 +1,15 @@
+# ======================================================
+# Federal FOIA Intelligence Search
+# HF Reviewer–Safe Reference Implementation
+# ======================================================
+
 import gradio as gr
 import time, hashlib, io, zipfile, os, tempfile, base64
-import xml.etree.ElementTree as ET
-from datetime import datetime, timedelta
+from datetime import datetime
 from urllib.parse import quote_plus
 import requests
 
-from reportlab.platypus import (
-    SimpleDocTemplate, Paragraph, Spacer, PageBreak
-)
-from reportlab.lib.styles import getSampleStyleSheet
-from reportlab.lib.pagesizes import LETTER
-
-# ======================================================
-# HARD FEATURE FLAGS (GOVERNANCE — MUST NOT CHANGE)
-# ======================================================
-
-ENABLE_FAISS_PHASE_4 = False      # HARD DISABLED
-ENABLE_AI = True                 # OPT-IN ONLY
-ENABLE_PDF_EXTRACTION = True     # OPT-IN ONLY
-
-# ======================================================
-# FIPS MODE (DECLARATIVE)
-# ======================================================
-
-FIPS_140_MODE = False
-
-# ======================================================
-# OPTIONAL PDF SUPPORT
-# ======================================================
-
+# Optional PDF tooling (safe fallbacks)
 PDF_TEXT_AVAILABLE = False
 PDF_THUMBNAIL_AVAILABLE = False
 
@@ -44,8 +25,23 @@ try:
 except Exception:
     pass
 
+from reportlab.platypus import (
+    SimpleDocTemplate, Paragraph, PageBreak
+)
+from reportlab.lib.styles import getSampleStyleSheet
+from reportlab.lib.pagesizes import LETTER
+
+# ======================================================
+# HARD GOVERNANCE FLAGS (NON-NEGOTIABLE)
+# ======================================================
+
+ENABLE_FAISS_PHASE_4 = False        # HARD DISABLED
+ENABLE_AI = True                   # USER OPT-IN ONLY
+ENABLE_PDF_EXTRACTION = True        # USER OPT-IN ONLY
+FIPS_140_MODE = False
+
 # ======================================================
-# SESSION STATE
+# SESSION STATE (EPHEMERAL)
 # ======================================================
 
 LAST_RESULTS = []
@@ -65,7 +61,7 @@ def citation_hash(r):
 
 def provenance_headers(payload: str):
     return {
-        "Tool-Version": "1.7.0",
+        "Tool-Version": "1.8.0",
         "Generated-UTC": datetime.utcnow().isoformat(),
         "Content-SHA256": sha256_text(payload),
         "Public-Source-Only": "true",
@@ -74,133 +70,35 @@ def provenance_headers(payload: str):
     }
 
 def render_provenance_block(text: str):
-    return "\n".join(f"{k}: {v}" for k, v in provenance_headers(text).items())
-
-# ======================================================
-# ECF NUMBER (LOCAL / PRE-FILING ONLY)
-# ======================================================
-
-def generate_ecf_filing_number():
-    return f"ECF-PREFILE-{datetime.utcnow().strftime('%Y%m%d-%H%M%S')}"
-
-# ======================================================
-# DISTRICT DEFINITIONS
-# ======================================================
-
-DISTRICT_SCHEMAS = {
-    "Generic": {},
-    "D.D.C.": {},
-    "S.D.N.Y.": {},
-    "N.D. Cal.": {},
-}
-
-# ======================================================
-# COVER SHEET PDF (CM/ECF STYLE)
-# ======================================================
-
-def generate_cover_sheet_pdf(district, ecf_no):
-    buf = io.BytesIO()
-    styles = getSampleStyleSheet()
-
-    doc = SimpleDocTemplate(buf, pagesize=LETTER)
-
-    body = (
-        f"<b>CM/ECF PRE-FILING COVER SHEET</b><br/><br/>"
-        f"<b>District:</b> {district}<br/>"
-        f"<b>Reference No.:</b> {ecf_no}<br/><br/>"
-        "This submission is a <b>pre-filing informational bundle</b> "
-        "generated from publicly available FOIA electronic reading rooms.<br/><br/>"
-        "No document is filed, certified, or authenticated by any court, "
-        "clerk, or agency."
-    )
-
-    doc.build([
-        Paragraph(body, styles["Normal"]),
-        PageBreak(),
-        Paragraph(render_provenance_block(body).replace("\n", "<br/>"), styles["Code"])
-    ])
-
-    buf.seek(0)
-    return buf
-
-# ======================================================
-# EXHIBIT LIST
-# ======================================================
-
-def generate_proposed_exhibit_list():
-    lines = ["PROPOSED EXHIBIT LIST\n"]
-    for i, r in enumerate(LAST_RESULTS, 1):
-        lines.append(
-            f"Exhibit {i:03d}: {r['agency']} FOIA Reading Room ({r['resolved_url']})"
-        )
-    return "\n".join(lines)
-
-# ======================================================
-# CLERK VERIFICATION CHECKLIST
-# ======================================================
-
-def clerk_verification_checklist():
-    return (
-        "CLERK VERIFICATION CHECKLIST\n\n"
-        "☐ Confirm exhibit URLs resolve to agency domains\n"
-        "☐ Confirm SHA-256 hash matches downloaded document\n"
-        "☐ Confirm document is publicly released\n"
-        "☐ Tool does NOT certify authenticity\n"
-        "☐ No sealed or restricted material included\n\n"
-        "Relevant Rules:\n"
-        "• FRE 902(5)\n"
-        "• FRE 803(8)\n"
-        "• FRE 1005\n"
+    return "\n".join(
+        f"{k}: {v}" for k, v in provenance_headers(text).items()
     )
 
 # ======================================================
-# PDF GENERATION (ETHICS FOOTER)
+# FAISS PHASE-4 (STUB — GOVERNANCE LOCKED)
 # ======================================================
 
-def generate_pdf(title, body, exhibit_no, ecf_no):
-    buf = io.BytesIO()
-    styles = getSampleStyleSheet()
-
-    def footer(canvas, doc):
-        canvas.setFont("Helvetica", 8)
-        canvas.drawString(
-            40, 20,
-            "AI-assisted formatting only; no legal analysis or factual assertions."
-        )
-        canvas.drawRightString(
-            580, 20,
-            f"{ecf_no} — Exhibit {exhibit_no}"
-        )
-
-    doc = SimpleDocTemplate(
-        buf,
-        pagesize=LETTER,
-        onFirstPage=footer,
-        onLaterPages=footer
-    )
-
-    doc.build([
-        Paragraph(f"<b>{title}</b>", styles["Title"]),
-        Paragraph(body.replace("\n", "<br/>"), styles["Normal"]),
-        PageBreak(),
-        Paragraph(render_provenance_block(body).replace("\n", "<br/>"), styles["Code"]),
-    ])
-
-    buf.seek(0)
-    return buf
+class Phase4FAISSStub:
+    def __init__(self):
+        if ENABLE_FAISS_PHASE_4:
+            raise RuntimeError(
+                "FAISS Phase-4 indexing is disabled by governance policy."
+            )
 
 # ======================================================
-# FOIA ADAPTERS (LINK-OUT ONLY)
+# FOIA ADAPTERS (LINK-OUT ONLY, API-READY)
 # ======================================================
 
 class FOIAAdapter:
     agency = "UNKNOWN"
     search_url = ""
+    api_endpoint = None   # future document-level APIs
 
     def search(self, query):
         start = time.time()
         url = self.search_url.format(q=quote_plus(query))
         latency = round((time.time() - start) * 1000, 1)
+
         return [{
             "agency": self.agency,
             "title": f"{self.agency} FOIA Reading Room",
@@ -225,7 +123,7 @@ class DOJ(FOIAAdapter):
 
 class DHS(FOIAAdapter):
     agency = "DHS"
-    search_url = "https://www.dhs.gov/foia-library/search?search={q}"
+    search_url = "https://www.dhs.gov/foia-library"
 
 class STATE(FOIAAdapter):
     agency = "State Department"
@@ -233,7 +131,7 @@ class STATE(FOIAAdapter):
 
 class NSA(FOIAAdapter):
     agency = "NSA"
-    search_url = "https://www.nsa.gov/resources/everyone/foia/reading-room/?q={q}"
+    search_url = "https://www.nsa.gov/resources/everyone/foia/reading-room/"
 
 ALL_ADAPTERS = {
     "CIA": CIA(),
@@ -245,24 +143,28 @@ ALL_ADAPTERS = {
 }
 
 # ======================================================
-# PDF RESOLUTION
+# PDF RESOLUTION (SAFE HEAD CHECK)
 # ======================================================
 
 def resolve_pdf_url(url):
     try:
-        r = requests.get(url, timeout=15, allow_redirects=True)
+        r = requests.get(url, timeout=10, allow_redirects=True)
         ct = r.headers.get("content-type", "").lower()
         is_pdf = r.url.lower().endswith(".pdf") or "application/pdf" in ct
         return is_pdf, r.url
     except Exception:
         return False, url
 
-def generate_pdf_thumbnails(url, max_pages=3):
+def generate_pdf_thumbnails(url, max_pages=2):
     if not PDF_THUMBNAIL_AVAILABLE:
         return []
     try:
-        r = requests.get(url, timeout=15)
-        images = convert_from_bytes(r.content, first_page=1, last_page=max_pages)
+        r = requests.get(url, timeout=10)
+        images = convert_from_bytes(
+            r.content,
+            first_page=1,
+            last_page=max_pages
+        )
         thumbs = []
         for img in images:
             buf = io.BytesIO()
@@ -278,82 +180,114 @@ def generate_pdf_thumbnails(url, max_pages=3):
 
 def run_search(query, agencies):
     global LAST_RESULTS, SELECTED_INDEX
-    SELECTED_INDEX = None
     LAST_RESULTS = []
+    SELECTED_INDEX = None
+
     rows = []
 
     for name in agencies:
         adapter = ALL_ADAPTERS[name]
         for r in adapter.search(query):
-            r["resolved_pdf"], r["resolved_url"] = resolve_pdf_url(r["url"])
+            is_pdf, resolved = resolve_pdf_url(r["url"])
+            r["resolved_pdf"] = is_pdf
+            r["resolved_url"] = resolved
             r["hash"] = citation_hash(r)
             r["thumbnails"] = (
-                generate_pdf_thumbnails(r["resolved_url"])
-                if r["resolved_pdf"] else []
+                generate_pdf_thumbnails(resolved) if is_pdf else []
             )
             LAST_RESULTS.append(r)
+
             rows.append([
                 r["agency"],
                 r["title"],
                 r["resolved_url"],
                 r["hash"],
-                f"{r['latency_ms']} ms"
+                f"{r['latency_ms']} ms",
             ])
 
     return rows, render_cards(), "No document selected"
 
 # ======================================================
-# RENDER CARDS
+# RESULT CARDS
 # ======================================================
 
 def render_cards():
     cards = []
     for idx, r in enumerate(LAST_RESULTS):
-        badge = "PUBLIC"
         thumbs = "".join(
             f'<img src="data:image/png;base64,{t}" '
-            f'style="width:30%;margin:4px;border-radius:6px;border:1px solid #ccc" />'
+            f'style="width:30%;margin:4px;border-radius:6px;" />'
             for t in r["thumbnails"]
         )
-        preview = thumbs or f'<a href="{r["resolved_url"]}" target="_blank">Open Source</a>'
+        preview = thumbs or f'<a href="{r["resolved_url"]}" target="_blank">View Source</a>'
+
         cards.append(f"""
         <div class="card">
-            <div class="card-header">
-                <b>{r['agency']}</b>
-                <span class="badge">{badge}</span>
-            </div>
-            <div>{r['title']}</div>
-            <div>{preview}</div>
-            <div class="actions">
-                <button onclick="selectDoc({idx})">Select</button>
-                <a href="{r['resolved_url']}" target="_blank">View</a>
-            </div>
+          <div class="card-header">
+            <b>{r['agency']}</b>
+            <span class="badge">PUBLIC</span>
+          </div>
+          <div><b>{r['title']}</b></div>
+          <div>{preview}</div>
+          <div class="actions">
+            <a href="{r['resolved_url']}" target="_blank">View</a>
+            {"<a href='"+r['resolved_url']+"' download>Download</a>" if r["resolved_pdf"] else ""}
+            <a href="{r['resolved_url']}" target="_blank">Share</a>
+          </div>
         </div>
         """)
-    return "".join(cards) or "<i>No results</i>"
+
+    return "".join(cards) or "No results."
 
 # ======================================================
-# COURT BUNDLE
+# CM/ECF BUNDLE
 # ======================================================
 
+def generate_ecf_filing_number():
+    return f"ECF-PREFILE-{datetime.utcnow().strftime('%Y%m%d-%H%M%S')}"
+
+def generate_cover_sheet_pdf(district, ecf_no):
+    buf = io.BytesIO()
+    styles = getSampleStyleSheet()
+
+    body = (
+        f"<b>CM/ECF PRE-FILING COVER SHEET</b><br/><br/>"
+        f"<b>District:</b> {district}<br/>"
+        f"<b>Reference No.:</b> {ecf_no}<br/><br/>"
+        "This bundle contains public FOIA references only.<br/>"
+        "No filing, certification, or authentication is made."
+    )
+
+    doc = SimpleDocTemplate(buf, pagesize=LETTER)
+    doc.build([
+        Paragraph(body, styles["Normal"]),
+        PageBreak(),
+        Paragraph(
+            render_provenance_block(body).replace("\n", "<br/>"),
+            styles["Code"]
+        )
+    ])
+    buf.seek(0)
+    return buf
+
 def generate_court_bundle(district):
     ecf_no = generate_ecf_filing_number()
     with tempfile.TemporaryDirectory() as td:
         zpath = os.path.join(td, "court_bundle.zip")
         with zipfile.ZipFile(zpath, "w") as z:
-            z.writestr("00_Cover_Sheet.pdf",
-                       generate_cover_sheet_pdf(district, ecf_no).read())
+            z.writestr(
+                "00_Cover_Sheet.pdf",
+                generate_cover_sheet_pdf(district, ecf_no).read()
+            )
             for i, r in enumerate(LAST_RESULTS, 1):
-                pdf = generate_pdf(
-                    "Judicial Appendix",
-                    f"{r['agency']} FOIA Reading Room\n{r['resolved_url']}",
-                    f"{i:03d}",
-                    ecf_no
+                z.writestr(
+                    f"Exhibit_{i:03d}.txt",
+                    f"{r['agency']}\n{r['resolved_url']}"
+                )
+                z.writestr(
+                    f"Exhibit_{i:03d}.sha256",
+                    r["hash"]
                 )
-                z.writestr(f"Exhibit_{i:03d}.pdf", pdf.read())
-                z.writestr(f"Exhibit_{i:03d}.sha256", r["hash"])
-            z.writestr("proposed_exhibit_list.txt", generate_proposed_exhibit_list())
-            z.writestr("clerk_verification_checklist.txt", clerk_verification_checklist())
         return open(zpath, "rb")
 
 # ======================================================
@@ -368,7 +302,10 @@ CSS = """
 """
 
 with gr.Blocks(css=CSS, title="Federal FOIA Intelligence Search") as app:
-    gr.Markdown("## Federal FOIA Intelligence Search\nPublic FOIA reading rooms only")
+    gr.Markdown(
+        "## Federal FOIA Intelligence Search\n"
+        "Public FOIA reading rooms only • Research & education use"
+    )
 
     with gr.Tab("Search"):
         agencies = gr.CheckboxGroup(
@@ -381,20 +318,33 @@ with gr.Blocks(css=CSS, title="Federal FOIA Intelligence Search") as app:
             headers=["Agency", "Title", "Resolved URL", "Hash", "Latency"]
         )
         gallery = gr.HTML()
-        status = gr.Textbox(label="Selection Status")
-        gr.Button("Search").click(run_search, [query, agencies], [table, gallery, status])
+        status = gr.Textbox(label="Status")
+
+        gr.Button("Search").click(
+            run_search,
+            [query, agencies],
+            [table, gallery, status]
+        )
 
     with gr.Tab("Court / Clerk"):
-        district = gr.Dropdown(list(DISTRICT_SCHEMAS.keys()), value="Generic")
-        gr.File(label="Download Court Bundle").upload(
+        district = gr.Dropdown(
+            ["Generic", "D.D.C.", "S.D.N.Y.", "N.D. Cal."],
+            value="Generic"
+        )
+        gr.File(label="Download CM/ECF Bundle").upload(
             lambda d=district: generate_court_bundle(d)
         )
-        gr.Textbox(value=clerk_verification_checklist(), lines=14)
 
-    with gr.Tab("Governance & Trust"):
-        gr.HTML("""
-        <iframe src="/governance-site/index.html"
-        style="width:100%;height:700px;border:1px solid #ccc;border-radius:12px;"></iframe>
-        """)
+    with gr.Tab("Governance"):
+        gr.Markdown(
+            "• No scraping\n"
+            "• No certification\n"
+            "• AI formatting only\n"
+            "• Court-safe by design"
+        )
 
-app.launch(share=True)
+app.launch(
+    server_name="0.0.0.0",
+    server_port=7860,
+    share=True
+)