Update app.py

app.py

@@ -1,31 +1,55 @@
 import gradio as gr
-import time, hashlib, io, zipfile, os, tempfile
+import time, hashlib, io, zipfile, os, tempfile, base64
 import xml.etree.ElementTree as ET
-from datetime import datetime
+from datetime import datetime, timedelta
 from urllib.parse import quote_plus
+import requests
 
-from reportlab.platypus import SimpleDocTemplate, Paragraph, PageBreak
+from reportlab.platypus import (
+    SimpleDocTemplate, Paragraph, Spacer, PageBreak
+)
 from reportlab.lib.styles import getSampleStyleSheet
 from reportlab.lib.pagesizes import LETTER
 
 # ======================================================
-# HARD FEATURE FLAGS (GOVERNANCE ENFORCED)
+# HARD FEATURE FLAGS (GOVERNANCE — MUST NOT CHANGE)
 # ======================================================
 
-ENABLE_FAISS_PHASE_4 = False
-ENABLE_AI = True
+ENABLE_FAISS_PHASE_4 = False      # HARD DISABLED
+ENABLE_AI = True                 # OPT-IN ONLY
+ENABLE_PDF_EXTRACTION = True     # OPT-IN ONLY
 
 # ======================================================
-# FIPS MODE
+# FIPS MODE (DECLARATIVE)
 # ======================================================
 
 FIPS_140_MODE = False
 
+# ======================================================
+# OPTIONAL PDF SUPPORT
+# ======================================================
+
+PDF_TEXT_AVAILABLE = False
+PDF_THUMBNAIL_AVAILABLE = False
+
+try:
+    from pdfminer.high_level import extract_text
+    PDF_TEXT_AVAILABLE = True
+except Exception:
+    pass
+
+try:
+    from pdf2image import convert_from_bytes
+    PDF_THUMBNAIL_AVAILABLE = True
+except Exception:
+    pass
+
 # ======================================================
 # SESSION STATE
 # ======================================================
 
 LAST_RESULTS = []
+SELECTED_INDEX = None
 
 # ======================================================
 # CRYPTOGRAPHIC CORE
@@ -34,6 +58,11 @@ LAST_RESULTS = []
 def sha256_text(t: str):
     return hashlib.sha256(t.encode()).hexdigest()
 
+def citation_hash(r):
+    return hashlib.sha256(
+        f"{r['agency']}|{r['resolved_url']}|{r['timestamp']}".encode()
+    ).hexdigest()[:16]
+
 def provenance_headers(payload: str):
     return {
         "Tool-Version": "1.7.0",
@@ -66,7 +95,7 @@ DISTRICT_SCHEMAS = {
 }
 
 # ======================================================
-# COVER SHEET PDF
+# COVER SHEET PDF (CM/ECF STYLE)
 # ======================================================
 
 def generate_cover_sheet_pdf(district, ecf_no):
@@ -79,10 +108,10 @@ def generate_cover_sheet_pdf(district, ecf_no):
         f"<b>CM/ECF PRE-FILING COVER SHEET</b><br/><br/>"
         f"<b>District:</b> {district}<br/>"
         f"<b>Reference No.:</b> {ecf_no}<br/><br/>"
-        "This submission is a <b>pre-filing informational bundle</b> generated "
-        "from publicly available FOIA electronic reading rooms.<br/><br/>"
-        "No document in this bundle is filed, certified, or authenticated "
-        "by any court, clerk, or agency."
+        "This submission is a <b>pre-filing informational bundle</b> "
+        "generated from publicly available FOIA electronic reading rooms.<br/><br/>"
+        "No document is filed, certified, or authenticated by any court, "
+        "clerk, or agency."
     )
 
     doc.build([
@@ -102,8 +131,7 @@ def generate_proposed_exhibit_list():
     lines = ["PROPOSED EXHIBIT LIST\n"]
     for i, r in enumerate(LAST_RESULTS, 1):
         lines.append(
-            f"Exhibit {i:03d}: {r['agency']} FOIA Reading Room "
-            f"({r['url']})"
+            f"Exhibit {i:03d}: {r['agency']} FOIA Reading Room ({r['resolved_url']})"
         )
     return "\n".join(lines)
 
@@ -114,11 +142,11 @@ def generate_proposed_exhibit_list():
 def clerk_verification_checklist():
     return (
         "CLERK VERIFICATION CHECKLIST\n\n"
-        "☐ Confirm exhibit URLs resolve to issuing agency domains\n"
-        "☐ Confirm SHA-256 hash matches downloaded agency document\n"
+        "☐ Confirm exhibit URLs resolve to agency domains\n"
+        "☐ Confirm SHA-256 hash matches downloaded document\n"
         "☐ Confirm document is publicly released\n"
-        "☐ Note: Tool does NOT certify authenticity\n"
-        "☐ Note: No sealed or restricted material included\n\n"
+        "☐ Tool does NOT certify authenticity\n"
+        "☐ No sealed or restricted material included\n\n"
         "Relevant Rules:\n"
         "• FRE 902(5)\n"
         "• FRE 803(8)\n"
@@ -126,7 +154,7 @@ def clerk_verification_checklist():
     )
 
 # ======================================================
-# PDF GENERATION (WITH AI / ETHICS FOOTER)
+# PDF GENERATION (ETHICS FOOTER)
 # ======================================================
 
 def generate_pdf(title, body, exhibit_no, ecf_no):
@@ -137,7 +165,7 @@ def generate_pdf(title, body, exhibit_no, ecf_no):
         canvas.setFont("Helvetica", 8)
         canvas.drawString(
             40, 20,
-            "AI-assisted formatting only; no substantive analysis or factual assertions."
+            "AI-assisted formatting only; no legal analysis or factual assertions."
         )
         canvas.drawRightString(
             580, 20,
@@ -162,43 +190,147 @@ def generate_pdf(title, body, exhibit_no, ecf_no):
     return buf
 
 # ======================================================
-# FOIA ADAPTERS
+# FOIA ADAPTERS (LINK-OUT ONLY)
 # ======================================================
 
 class FOIAAdapter:
-    agency = ""
-    url = ""
-    def search(self, q):
+    agency = "UNKNOWN"
+    search_url = ""
+
+    def search(self, query):
+        start = time.time()
+        url = self.search_url.format(q=quote_plus(query))
+        latency = round((time.time() - start) * 1000, 1)
         return [{
             "agency": self.agency,
             "title": f"{self.agency} FOIA Reading Room",
-            "url": self.url.format(q=quote_plus(q)),
+            "url": url,
+            "timestamp": datetime.utcnow().isoformat(),
+            "latency_ms": latency,
+            "sealed": False,
+            "redacted": False,
         }]
 
 class CIA(FOIAAdapter):
     agency = "CIA"
-    url = "https://www.cia.gov/readingroom/search/site/{q}"
+    search_url = "https://www.cia.gov/readingroom/search/site/{q}"
 
 class FBI(FOIAAdapter):
     agency = "FBI"
-    url = "https://vault.fbi.gov/search?SearchableText={q}"
+    search_url = "https://vault.fbi.gov/search?SearchableText={q}"
+
+class DOJ(FOIAAdapter):
+    agency = "DOJ"
+    search_url = "https://www.justice.gov/foia/library?search={q}"
+
+class DHS(FOIAAdapter):
+    agency = "DHS"
+    search_url = "https://www.dhs.gov/foia-library/search?search={q}"
+
+class STATE(FOIAAdapter):
+    agency = "State Department"
+    search_url = "https://foia.state.gov/Search/Search.aspx?q={q}"
+
+class NSA(FOIAAdapter):
+    agency = "NSA"
+    search_url = "https://www.nsa.gov/resources/everyone/foia/reading-room/?q={q}"
+
+ALL_ADAPTERS = {
+    "CIA": CIA(),
+    "FBI": FBI(),
+    "DOJ": DOJ(),
+    "DHS": DHS(),
+    "State": STATE(),
+    "NSA": NSA(),
+}
 
-ALL_ADAPTERS = {"CIA": CIA(), "FBI": FBI()}
+# ======================================================
+# PDF RESOLUTION
+# ======================================================
+
+def resolve_pdf_url(url):
+    try:
+        r = requests.get(url, timeout=15, allow_redirects=True)
+        ct = r.headers.get("content-type", "").lower()
+        is_pdf = r.url.lower().endswith(".pdf") or "application/pdf" in ct
+        return is_pdf, r.url
+    except Exception:
+        return False, url
+
+def generate_pdf_thumbnails(url, max_pages=3):
+    if not PDF_THUMBNAIL_AVAILABLE:
+        return []
+    try:
+        r = requests.get(url, timeout=15)
+        images = convert_from_bytes(r.content, first_page=1, last_page=max_pages)
+        thumbs = []
+        for img in images:
+            buf = io.BytesIO()
+            img.save(buf, format="PNG")
+            thumbs.append(base64.b64encode(buf.getvalue()).decode())
+        return thumbs
+    except Exception:
+        return []
 
 # ======================================================
 # SEARCH
 # ======================================================
 
 def run_search(query, agencies):
-    global LAST_RESULTS
+    global LAST_RESULTS, SELECTED_INDEX
+    SELECTED_INDEX = None
     LAST_RESULTS = []
     rows = []
-    for a in agencies:
-        for r in ALL_ADAPTERS[a].search(query):
-            r["hash"] = sha256_text(r["url"])
+
+    for name in agencies:
+        adapter = ALL_ADAPTERS[name]
+        for r in adapter.search(query):
+            r["resolved_pdf"], r["resolved_url"] = resolve_pdf_url(r["url"])
+            r["hash"] = citation_hash(r)
+            r["thumbnails"] = (
+                generate_pdf_thumbnails(r["resolved_url"])
+                if r["resolved_pdf"] else []
+            )
             LAST_RESULTS.append(r)
-            rows.append([r["agency"], r["title"], r["url"], r["hash"]])
-    return rows
+            rows.append([
+                r["agency"],
+                r["title"],
+                r["resolved_url"],
+                r["hash"],
+                f"{r['latency_ms']} ms"
+            ])
+
+    return rows, render_cards(), "No document selected"
+
+# ======================================================
+# RENDER CARDS
+# ======================================================
+
+def render_cards():
+    cards = []
+    for idx, r in enumerate(LAST_RESULTS):
+        badge = "PUBLIC"
+        thumbs = "".join(
+            f'<img src="data:image/png;base64,{t}" '
+            f'style="width:30%;margin:4px;border-radius:6px;border:1px solid #ccc" />'
+            for t in r["thumbnails"]
+        )
+        preview = thumbs or f'<a href="{r["resolved_url"]}" target="_blank">Open Source</a>'
+        cards.append(f"""
+        <div class="card">
+            <div class="card-header">
+                <b>{r['agency']}</b>
+                <span class="badge">{badge}</span>
+            </div>
+            <div>{r['title']}</div>
+            <div>{preview}</div>
+            <div class="actions">
+                <button onclick="selectDoc({idx})">Select</button>
+                <a href="{r['resolved_url']}" target="_blank">View</a>
+            </div>
+        </div>
+        """)
+    return "".join(cards) or "<i>No results</i>"
 
 # ======================================================
 # COURT BUNDLE
@@ -206,63 +338,63 @@ def run_search(query, agencies):
 
 def generate_court_bundle(district):
     ecf_no = generate_ecf_filing_number()
-
     with tempfile.TemporaryDirectory() as td:
         zpath = os.path.join(td, "court_bundle.zip")
-
         with zipfile.ZipFile(zpath, "w") as z:
-            # Cover sheet
-            cover = generate_cover_sheet_pdf(district, ecf_no)
-            z.writestr("00_Cover_Sheet.pdf", cover.read())
-
-            # Exhibits
+            z.writestr("00_Cover_Sheet.pdf",
+                       generate_cover_sheet_pdf(district, ecf_no).read())
             for i, r in enumerate(LAST_RESULTS, 1):
                 pdf = generate_pdf(
                     "Judicial Appendix",
-                    f"{r['agency']} FOIA Reading Room\n{r['url']}",
+                    f"{r['agency']} FOIA Reading Room\n{r['resolved_url']}",
                     f"{i:03d}",
-                    ecf_no,
+                    ecf_no
                 )
                 z.writestr(f"Exhibit_{i:03d}.pdf", pdf.read())
-                z.writestr(
-                    f"Exhibit_{i:03d}.sha256",
-                    sha256_text(r["url"])
-                )
-
-            # Support docs
+                z.writestr(f"Exhibit_{i:03d}.sha256", r["hash"])
             z.writestr("proposed_exhibit_list.txt", generate_proposed_exhibit_list())
             z.writestr("clerk_verification_checklist.txt", clerk_verification_checklist())
-
         return open(zpath, "rb")
 
 # ======================================================
 # UI
 # ======================================================
 
-with gr.Blocks(title="Federal FOIA Intelligence Search") as app:
-    gr.Markdown("## Federal FOIA Intelligence Search")
+CSS = """
+.card { border:1px solid #ddd; border-radius:16px; padding:16px; margin-bottom:20px; }
+.card-header { display:flex; justify-content:space-between; }
+.actions { margin-top:10px; display:flex; gap:12px; }
+.badge { background:#eef; padding:4px 10px; border-radius:999px; }
+"""
+
+with gr.Blocks(css=CSS, title="Federal FOIA Intelligence Search") as app:
+    gr.Markdown("## Federal FOIA Intelligence Search\nPublic FOIA reading rooms only")
 
     with gr.Tab("Search"):
         agencies = gr.CheckboxGroup(
             list(ALL_ADAPTERS.keys()),
-            value=list(ALL_ADAPTERS.keys())
+            value=list(ALL_ADAPTERS.keys()),
+            label="Agencies"
         )
-        query = gr.Textbox()
+        query = gr.Textbox(placeholder="Search terms")
         table = gr.Dataframe(
-            headers=["Agency", "Title", "URL", "SHA-256"]
+            headers=["Agency", "Title", "Resolved URL", "Hash", "Latency"]
         )
-        gr.Button("Search").click(run_search, [query, agencies], table)
+        gallery = gr.HTML()
+        status = gr.Textbox(label="Selection Status")
+        gr.Button("Search").click(run_search, [query, agencies], [table, gallery, status])
 
-    with gr.Tab("Court Bundle"):
-        district = gr.Dropdown(
-            list(DISTRICT_SCHEMAS.keys()),
-            value="Generic"
-        )
+    with gr.Tab("Court / Clerk"):
+        district = gr.Dropdown(list(DISTRICT_SCHEMAS.keys()), value="Generic")
         gr.File(label="Download Court Bundle").upload(
             lambda d=district: generate_court_bundle(d)
         )
+        gr.Textbox(value=clerk_verification_checklist(), lines=14)
 
-    with gr.Tab("Clerk Checklist"):
-        gr.Textbox(value=clerk_verification_checklist(), lines=16)
+    with gr.Tab("Governance & Trust"):
+        gr.HTML("""
+        <iframe src="/governance-site/index.html"
+        style="width:100%;height:700px;border:1px solid #ccc;border-radius:12px;"></iframe>
+        """)
 
 app.launch()