Update app.py

app.py

@@ -1,25 +1,42 @@
 import gradio as gr
-import hashlib
 import time
+import hashlib
+import zipfile
+import io
 from datetime import datetime
-from urllib.parse import quote_plus
-from sentence_transformers import SentenceTransformer
-import faiss
-from reportlab.lib.pagesizes import LETTER
-from reportlab.pdfgen import canvas
-import tempfile
-import os
-import webbrowser
-
-# =====================================================
-# CONFIG
-# =====================================================
-
-SEMANTIC_MODEL = "all-MiniLM-L6-v2"
-
-# =====================================================
+from urllib.parse import quote_plus, urlparse
+from collections import defaultdict, Counter
+
+import plotly.graph_objects as go
+from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer
+from reportlab.lib.styles import getSampleStyleSheet
+
+# ======================================================
+# OPTIONAL SEMANTIC DEPENDENCIES (SAFE-GUARDED)
+# ======================================================
+
+FAISS_AVAILABLE = False
+try:
+    import faiss
+    from sentence_transformers import SentenceTransformer
+    FAISS_AVAILABLE = True
+except Exception:
+    FAISS_AVAILABLE = False
+
+# ======================================================
+# CONFIG / FEATURE GATES
+# ======================================================
+
+ENABLE_SEMANTIC = False          # Opt-in only
+ENABLE_PDF_EXPORT = True
+ENABLE_HEATMAP = True
+ENABLE_PDF_THUMBNAILS = True
+ENABLE_ENTITY_GRAPHS = True
+ENABLE_TIMELINES = True
+
+# ======================================================
 # BASE ADAPTER
-# =====================================================
+# ======================================================
 
 class FOIAAdapter:
     agency = "UNKNOWN"
@@ -33,16 +50,16 @@ class FOIAAdapter:
 
         return [{
             "agency": self.agency,
-            "title": f"{self.agency} FOIA Reading Room Result",
+            "title": f"{self.agency} FOIA Search Results",
             "url": url,
             "latency": latency,
             "is_live": self.is_live,
             "timestamp": datetime.utcnow().isoformat()
         }]
 
-# =====================================================
-# LIVE AGENCIES (PUBLIC FOIA)
-# =====================================================
+# ======================================================
+# LIVE AGENCIES (LINK-OUT ONLY)
+# ======================================================
 
 class CIA(FOIAAdapter):
     agency = "CIA"
@@ -74,16 +91,15 @@ class NSA(FOIAAdapter):
 
 LIVE_ADAPTERS = [CIA(), FBI(), DOJ(), DHS(), STATE(), GSA(), NSA()]
 
-# =====================================================
-# STUB ADAPTERS (OPT-IN)
-# =====================================================
+# ======================================================
+# STUB ADAPTERS (NON-EXPORTABLE)
+# ======================================================
 
 class StubAdapter(FOIAAdapter):
     is_live = False
-    search_url = ""
-
     def __init__(self, agency):
         self.agency = agency
+        self.search_url = ""
 
     def search(self, query):
         return [{
@@ -105,30 +121,44 @@ STUB_ADAPTERS = [
     StubAdapter("Special Activities"),
 ]
 
-# =====================================================
+# ======================================================
 # UTILITIES
-# =====================================================
+# ======================================================
 
 def citation_hash(r):
     raw = f"{r['agency']}{r['url']}{r['timestamp']}"
     return hashlib.sha256(raw.encode()).hexdigest()[:16]
 
-def bluebook(r):
+def bluebook_full(r):
     return (
         f"{r['agency']}, {r['title']}, FOIA Electronic Reading Room, "
         f"{r['url']} (retrieved {datetime.utcnow().strftime('%b %d, %Y')})."
     )
 
-# =====================================================
-# SEARCH
-# =====================================================
+# ======================================================
+# GLOBAL STATE
+# ======================================================
+
+LAST_LIVE_RECORDS = []
+
+# ======================================================
+# SEARCH HANDLER
+# ======================================================
+
+def run_search(query, include_stubs, semantic_mode):
+    global LAST_LIVE_RECORDS
+    LAST_LIVE_RECORDS = []
 
-def run_search(query, include_stubs):
-    rows = []
     adapters = LIVE_ADAPTERS + (STUB_ADAPTERS if include_stubs else [])
+    rows = []
+    coverage = defaultdict(int)
+
+    for adapter in adapters:
+        for r in adapter.search(query):
+            coverage[r["agency"]] += 1
+            if r["is_live"]:
+                LAST_LIVE_RECORDS.append(r)
 
-    for a in adapters:
-        for r in a.search(query):
             rows.append([
                 r["agency"],
                 "LIVE" if r["is_live"] else "STUB",
@@ -136,107 +166,161 @@ def run_search(query, include_stubs):
                 r["url"],
                 r["latency"],
                 citation_hash(r) if r["is_live"] else "",
-                bluebook(r) if r["is_live"] else "Not exportable (STUB)",
-                r
+                bluebook_full(r) if r["is_live"] else "Not exportable (STUB)"
             ])
 
-    return rows
-
-# =====================================================
-# PREVIEW / THUMBNAIL
-# =====================================================
-
-def preview(row):
-    if not row:
-        return "<i>Select a result</i>"
-
-    url = row[3]
-    if isinstance(url, str) and url.lower().endswith(".pdf"):
-        return f"<iframe src='{url}' width='100%' height='520'></iframe>"
-
-    return "<i>No PDF preview available</i>"
-
-# =====================================================
-# ANALYZE WITH AI (FREE, LOCAL)
-# =====================================================
-
-model = None
+    gap_md = "### Coverage Gaps\n"
+    for agency in [a.agency for a in LIVE_ADAPTERS]:
+        if coverage.get(agency, 0) == 0:
+            gap_md += f"- ❌ **{agency}**: no public results found\n"
+
+    return rows, gap_md, build_pdf_thumbnail_gallery()
+
+# ======================================================
+# PDF THUMBNAIL GALLERY
+# ======================================================
+
+def build_pdf_thumbnail_gallery():
+    if not ENABLE_PDF_THUMBNAILS:
+        return "<i>PDF previews disabled</i>"
+
+    cards = []
+
+    for r in LAST_LIVE_RECORDS:
+        url = r["url"]
+        if not url or not url.lower().endswith(".pdf"):
+            continue
+
+        card = f"""
+        <div style="border:1px solid #ddd;border-radius:8px;padding:12px;margin-bottom:16px;">
+            <b>{r['agency']} — {r['title']}</b><br><br>
+            <iframe src="{url}" width="100%" height="220"></iframe>
+            <div style="margin-top:8px;">
+                <a href="{url}" target="_blank">View</a> |
+                <a href="{url}" download>Download</a> |
+                <a href="#" onclick="shareDoc('{url}')">Share</a> |
+                <a href="#" onclick="askAI('{r['agency']}', '{r['title']}', '{url}')">Ask AI</a>
+            </div>
+        </div>
+        """
+        cards.append(card)
+
+    return "".join(cards) if cards else "<i>No PDF documents found.</i>"
+
+# ======================================================
+# CLIENT JS
+# ======================================================
+
+JS_HELPERS = """
+<script>
+function shareDoc(url) {
+    if (navigator.share) {
+        navigator.share({ title: "FOIA Document", url: url });
+    } else {
+        navigator.clipboard.writeText(url);
+        alert("Link copied to clipboard");
+    }
+}
+
+function askAI(agency, title, url) {
+    alert(
+        "AI Summary (Preview)\\n\\n" +
+        "Agency: " + agency + "\\n" +
+        "Title: " + title + "\\n\\n" +
+        "Semantic analysis, entities, and citations can be enabled in opt-in AI mode."
+    );
+}
+</script>
+"""
+
+# ======================================================
+# ENTITY GRAPH + TIMELINE
+# ======================================================
+
+def build_entity_graph():
+    domains = Counter(urlparse(r["url"]).netloc for r in LAST_LIVE_RECORDS)
+    return go.Figure([go.Bar(x=list(domains.keys()), y=list(domains.values()))])
+
+def build_timeline():
+    dates = Counter(r["timestamp"][:10] for r in LAST_LIVE_RECORDS)
+    return go.Figure([go.Bar(x=list(dates.keys()), y=list(dates.values()))])
+
+# ======================================================
+# FOIA REQUEST GENERATOR
+# ======================================================
+
+def generate_foia_request(requester, description):
+    buffer = io.BytesIO()
+    doc = SimpleDocTemplate(buffer)
+    styles = getSampleStyleSheet()
+    story = []
+
+    story.append(Paragraph("<b>Freedom of Information Act Request</b>", styles["Title"]))
+    story.append(Spacer(1, 12))
+    story.append(Paragraph(f"<b>Requester:</b> {requester}", styles["Normal"]))
+    story.append(Spacer(1, 8))
+    story.append(Paragraph("<b>Description:</b>", styles["Normal"]))
+    story.append(Paragraph(description, styles["Normal"]))
+    story.append(Spacer(1, 12))
+
+    agencies = ", ".join(sorted({r["agency"] for r in LAST_LIVE_RECORDS}))
+    story.append(Paragraph(f"<b>Agencies:</b> {agencies}", styles["Normal"]))
+
+    doc.build(story)
+    buffer.seek(0)
+    return buffer
+
+# ======================================================
+# UI
+# ======================================================
 
-def analyze(row):
-    global model
-    if not row or row[1] != "LIVE":
-        return "Analysis unavailable."
+with gr.Blocks(title="Federal FOIA Intelligence Search") as app:
+    gr.HTML(JS_HELPERS)
 
-    if model is None:
-        model = SentenceTransformer(SEMANTIC_MODEL)
+    gr.Markdown(
+        """
+        # 🏛️ Federal FOIA Intelligence Search  
+        **Public FOIA Electronic Reading Rooms Only**
 
-    return (
-        f"AI Summary (Metadata-based):\n\n"
-        f"Agency: {row[0]}\n"
-        f"Title: {row[2]}\n"
-        f"Public URL: {row[3]}\n\n"
-        f"This document originates from a public FOIA reading room. "
-        f"Use the View or Download buttons to inspect the source."
+        ✔ LIVE = court-ready  
+        ⚠ STUB = informational only  
+        """
     )
 
-# =====================================================
-# COURT-READY PDF EXPORT
-# =====================================================
-
-def export_pdf(rows):
-    live = [r for r in rows if r[1] == "LIVE"]
-    if not live:
-        return None
-
-    fd, path = tempfile.mkstemp(suffix=".pdf")
-    os.close(fd)
-
-    c = canvas.Canvas(path, pagesize=LETTER)
-    width, height = LETTER
+    query = gr.Textbox(label="Search FOIA Libraries")
+    include_stubs = gr.Checkbox(label="Include Extended Coverage (STUB)", value=False)
+    semantic_toggle = gr.Checkbox(label="Enable Semantic Mode (Opt-In)", value=False)
 
-    for i, r in enumerate(live):
-        c.setFont("Helvetica-Bold", 12)
-        c.drawString(40, height - 40, f"Exhibit A-{i+1}")
-        c.setFont("Helvetica", 10)
-        c.drawString(40, height - 70, r[2])
-        c.drawString(40, height - 90, r[3])
-        c.drawString(40, 40, f"Integrity Hash: {r[5]}")
-        c.showPage()
+    search_btn = gr.Button("Search")
 
-    c.save()
-    return path
+    results = gr.Dataframe(
+        headers=["Agency","Type","Title","URL","Latency","Citation Hash","Citation"],
+        interactive=True
+    )
 
-# =====================================================
-# UI
-# =====================================================
+    gap_panel = gr.Markdown()
+    pdf_gallery = gr.HTML()
 
-with gr.Blocks(title="Federal FOIA Intelligence Search") as app:
-    gr.Markdown("""
-    # 🏛️ Federal FOIA Intelligence Search  
-    **Public Electronic Reading Rooms Only**
-    """)
+    search_btn.click(
+        run_search,
+        inputs=[query, include_stubs, semantic_toggle],
+        outputs=[results, gap_panel, pdf_gallery]
+    )
 
-    q = gr.Textbox(label="Search FOIA Libraries")
-    include_stubs = gr.Checkbox(label="Include Extended Coverage (STUB)", value=False)
-    semantic = gr.Checkbox(label="Enable Semantic Refine (Opt-In)", value=False)
+    gr.Markdown("## 📄 PDF Document Previews")
+    pdf_gallery.render()
 
-    btn = gr.Button("Search")
+    gr.Markdown("## FOIA Request Generator")
+    requester = gr.Textbox(label="Your Name / Organization")
+    description = gr.Textbox(label="Describe the records requested", lines=4)
 
-    table = gr.Dataframe(
-        headers=["Agency", "Type", "Title", "URL", "Latency", "Hash", "Citation", "_raw"],
-        interactive=True,
-        visible=True
+    gr.Button("Generate FOIA Request PDF").click(
+        generate_foia_request,
+        inputs=[requester, description],
+        outputs=gr.File()
     )
 
-    preview_panel = gr.HTML()
-    analysis = gr.Textbox(label="AI Analysis", lines=6)
-
-    export_btn = gr.Button("Export Court PDF (LIVE ONLY)")
-    export_file = gr.File()
-
-    btn.click(run_search, [q, include_stubs], table)
-    table.select(lambda e: preview(e.value), None, preview_panel)
-    table.select(lambda e: analyze(e.value), None, analysis)
-    export_btn.click(export_pdf, table, export_file)
+    gr.Button("Show Entity Graph").click(build_entity_graph, outputs=gr.Plot())
+    gr.Button("Show Timeline").click(build_timeline, outputs=gr.Plot())
 
 app.launch()