Update app.py

app.py

@@ -1,226 +1,131 @@
-# app.py
 # Federal FOIA Intelligence Search
 # Public Electronic Reading Rooms Only
 
 import gradio as gr
-import asyncio
 import time
-import csv
-import io
-import zipfile
 from collections import defaultdict
-from typing import Dict, List
+from typing import List, Dict
 
 from ingest.registry import get_all_adapters
-from policy.kill_switch import KillSwitch
-from analytics import log_event
-
+from analytics.semantic import semantic_cluster
+from analytics.health import adapter_health
+from analytics.foia_bcodes import classify_b_codes
+from exports.journalist_zip import build_zip
+from ui.heatmap import render_heatmap
 
 APP_TITLE = "Federal FOIA Intelligence Search"
-APP_SUBTITLE = "Public Electronic Reading Rooms Only"
+SUBTITLE = "Public Electronic Reading Rooms Only"
 
-kill_switch = KillSwitch()
 ALL_ADAPTERS = get_all_adapters()
 
 
-# ============================================================
-# Utility: FOIA Exemption Heuristic
-# ============================================================
-
-def classify_foia_exemption(text: str) -> str:
-    text = text.lower()
-    if "classified" in text or "national security" in text:
-        return "b(1)"
-    if "statute" in text:
-        return "b(3)"
-    if "privacy" in text or "personnel" in text:
-        return "b(6)"
-    if "law enforcement" in text or "investigation" in text:
-        return "b(7)"
-    return "—"
-
+def run_search(query: str, include_stubs: bool, extended: bool):
+    start = time.time()
 
-# ============================================================
-# Adapter Filtering
-# ============================================================
+    results = []
+    per_agency_counts = defaultdict(int)
+    health = {}
 
-def get_active_adapters(include_stubs, enable_extended):
-    active = {}
     for name, adapter in ALL_ADAPTERS.items():
-        if kill_switch.is_disabled(name):
+        if adapter.is_stub and not include_stubs:
             continue
-        if getattr(adapter, "is_stub", False) and not include_stubs:
+        if adapter.extended and not extended:
             continue
-        if getattr(adapter, "extended", False) and not enable_extended:
+        if adapter.killed:
             continue
-        active[name] = adapter
-    return active
-
 
-# ============================================================
-# Async Search + Metrics
-# ============================================================
-
-async def timed_search(adapter, query):
-    start = time.time()
-    try:
-        results = await adapter.search(query)
-        latency = round(time.time() - start, 2)
-        return results, latency, "🟢 Healthy"
-    except Exception:
-        return [], None, "🔴 Error"
-
-
-async def federated_search(query, adapters):
-    tasks = {
-        name: timed_search(adapter, query)
-        for name, adapter in adapters.items()
+        try:
+            t0 = time.time()
+            docs = adapter.search_sync(query)
+            latency = time.time() - t0
+            health[name] = adapter_health(latency)
+            for d in docs:
+                d["agency"] = name
+                d["b_codes"] = classify_b_codes(d)
+                results.append(d)
+                per_agency_counts[name] += 1
+        except Exception as e:
+            health[name] = {"status": "error", "detail": str(e)}
+
+    clusters = semantic_cluster(results)
+    elapsed = round(time.time() - start, 2)
+
+    return {
+        "results": results,
+        "clusters": clusters,
+        "counts": dict(per_agency_counts),
+        "health": health,
+        "elapsed": elapsed,
     }
 
-    responses = await asyncio.gather(*tasks.values())
-
-    results = []
-    metrics = {}
-
-    for (name, _), (docs, latency, health) in zip(tasks.items(), responses):
-        metrics[name] = {
-            "count": len(docs),
-            "latency": latency,
-            "health": health
-        }
-        for d in docs:
-            d["agency"] = name
-            d["foia_exemption"] = classify_foia_exemption(
-                d.get("snippet", "")
-            )
-            results.append(d)
-
-    return results, metrics
-
-
-# ============================================================
-# Search Handler
-# ============================================================
-
-def search_handler(query, include_stubs, enable_extended):
-    if not query.strip():
-        return [], [], "⚠️ Enter a search term."
 
-    log_event("search", {"query": query})
-
-    adapters = get_active_adapters(include_stubs, enable_extended)
-
-    try:
-        results, metrics = asyncio.run(
-            federated_search(query, adapters)
-        )
-    except RuntimeError:
-        loop = asyncio.get_event_loop()
-        results, metrics = loop.run_until_complete(
-            federated_search(query, adapters)
-        )
-
-    table = [
-        [
-            r["agency"],
-            r.get("title"),
-            r.get("url"),
-            r.get("snippet"),
-            r["foia_exemption"]
-        ]
-        for r in results
-    ]
-
-    heatmap = [
-        [
-            agency,
-            m["count"],
-            m["latency"] or "—",
-            m["health"]
-        ]
-        for agency, m in metrics.items()
-    ]
-
-    status = f"Found {len(results)} documents across {len(metrics)} agencies."
-    return table, heatmap, status
-
-
-# ============================================================
-# Journalist ZIP Export
-# ============================================================
-
-def export_selected(rows):
-    if not rows:
-        return None
-
-    buf = io.BytesIO()
-    with zipfile.ZipFile(buf, "w") as z:
-        csv_buf = io.StringIO()
-        writer = csv.writer(csv_buf)
-        writer.writerow(["Agency", "Title", "URL", "Snippet", "FOIA Exemption"])
-        for r in rows:
-            writer.writerow(r)
-        z.writestr("index.csv", csv_buf.getvalue())
-        z.writestr(
-            "README.txt",
-            "Federal FOIA Intelligence Search\nPublic Electronic Reading Rooms Only"
+def render_results(state):
+    md = []
+    for r in state["results"]:
+        badge = f"⚖️ {','.join(r['b_codes'])}" if r["b_codes"] else ""
+        md.append(
+            f"### [{r['title']}]({r['url']})\n"
+            f"*{r['agency']}* {badge}\n\n"
+            f"{r['snippet']}\n"
         )
-
-    buf.seek(0)
-    return buf
+    return "\n---\n".join(md)
 
 
-# ============================================================
-# UI
-# ============================================================
-
 with gr.Blocks(title=APP_TITLE) as demo:
-    gr.Markdown(f"""
-# 🏛️ **{APP_TITLE}**
-### *{APP_SUBTITLE}*
-
-Public-interest discovery across U.S. Government FOIA Electronic Reading Rooms.
-""")
+    gr.Markdown(f"# 🏛️ {APP_TITLE}\n### *{SUBTITLE}*")
+    gr.Markdown(
+        "Federated discovery across **lawfully published U.S. Government FOIA Electronic Reading Rooms**.\n\n"
+        "### Safeguards\n"
+        "- Public documents only\n"
+        "- Robots.txt enforced\n"
+        "- No restricted systems\n"
+        "- No classified access\n"
+        "- Per-agency kill switches\n"
+    )
 
-    query = gr.Textbox(label="Search FOIA Reading Rooms")
-    include_stubs = gr.Checkbox(label="Include Stub Sources", value=True)
-    enable_extended = gr.Checkbox(label="Enable Extended Features", value=False)
+    query = gr.Textbox(label="Search FOIA Reading Rooms", placeholder="UAP")
+    include_stubs = gr.Checkbox(True, label="Include Stub Sources (Coverage Only)")
+    extended = gr.Checkbox(False, label="Enable Extended Features (Live Intelligence-Related Rooms)")
 
     search_btn = gr.Button("Search")
-    status = gr.Markdown()
-
-    results = gr.Dataframe(
-        headers=["Agency", "Title", "URL", "Snippet", "FOIA Exemption"],
-        interactive=True,
-        wrap=True
-    )
 
-    coverage = gr.Dataframe(
-        headers=["Agency", "Result Count", "Latency (s)", "Health"],
-        interactive=False
-    )
+    results_md = gr.Markdown()
+    heatmap = gr.HTML()
+    latency_md = gr.Markdown()
+    cluster_md = gr.Markdown()
+    export_btn = gr.Button("🧾 Export Selected to ZIP")
+
+    state = gr.State()
+
+    def on_search(q, s, e):
+        state_val = run_search(q, s, e)
+        return (
+            state_val,
+            render_results(state_val),
+            render_heatmap(state_val["counts"]),
+            f"⏱️ Completed in {state_val['elapsed']}s",
+            f"🧠 Clusters: {len(state_val['clusters'])}",
+        )
 
     search_btn.click(
-        search_handler,
-        inputs=[query, include_stubs, enable_extended],
-        outputs=[results, coverage, status]
+        on_search,
+        inputs=[query, include_stubs, extended],
+        outputs=[state, results_md, heatmap, latency_md, cluster_md],
     )
 
-    export_btn = gr.Button("🧾 Export Selected Results (ZIP)")
-    export_file = gr.File()
-
     export_btn.click(
-        export_selected,
-        inputs=[results],
-        outputs=[export_file]
+        lambda s: build_zip(s["results"]),
+        inputs=[state],
+        outputs=[],
     )
 
-    gr.Markdown("""
----
-### ⚖️ Legal Notice
-FOIA exemption labels are **heuristic indicators only** and do not replace
-official agency determinations.
-""")
+    gr.Markdown(
+        "## ⚖️ Legal & Ethical Notice\n"
+        "This system indexes **only documents already released to the public** under FOIA.\n\n"
+        "**It does not:**\n"
+        "- Circumvent access controls\n"
+        "- Access classified systems\n"
+        "- Declassify or infer restricted information\n"
+    )
 
-if __name__ == "__main__":
-    demo.launch()
+demo.launch()