Create app.py

app.py

@@ -1,8 +1,15 @@
 import gradio as gr
 import time
 import re
-from typing import List, Dict
 from collections import defaultdict
+from typing import List, Dict
+
+from analytics import log_event
+from policy.kill_switch import KillSwitch
+from semantic.faiss_index import SemanticIndex
+from export.journalist_zip import export_zip
+from reports.explainability import explainability_report
+from ingest.registry import get_all_adapters
 
 ###############################################################################
 # GLOBAL STATE (HF SAFE)
@@ -10,9 +17,11 @@ from collections import defaultdict
 
 RESULT_CACHE: List[Dict] = []
 CURRENT_INDEX = 0
+SEMANTIC = SemanticIndex()
+KILL = KillSwitch()
 
 ###############################################################################
-# AGENCY POLICY / ROBOTS / LIVE SAFETY
+# AGENCY POLICY / ROBOTS
 ###############################################################################
 
 AGENCY_POLICY = {
@@ -21,31 +30,10 @@ AGENCY_POLICY = {
     "ODNI": {"robots": True,  "live": True},
     "USAF": {"robots": True,  "live": True},
     "NSA":  {"robots": False, "live": False},
-    "NRO":  {"robots": False, "live": False},
-    "SAP":  {"robots": False, "live": False},
 }
 
 ###############################################################################
-# KILL SWITCH (AUTO + MANUAL SAFE)
-###############################################################################
-
-class KillSwitch:
-    def __init__(self):
-        self.disabled = {}
-
-    def disable(self, agency: str, reason: str):
-        self.disabled[agency] = reason
-
-    def enabled(self, agency: str) -> bool:
-        return agency not in self.disabled
-
-    def reason(self, agency: str) -> str:
-        return self.disabled.get(agency, "")
-
-KILL = KillSwitch()
-
-###############################################################################
-# FOIA EXEMPTION CLASSIFIER
+# FOIA EXEMPTIONS
 ###############################################################################
 
 B_CODES = {
@@ -57,186 +45,166 @@ B_CODES = {
 
 def classify_exemptions(text: str) -> Dict:
     found = [k for k in B_CODES if k in text.lower()]
-    confidence = round(min(1.0, 0.3 * len(found)), 2)
-    return {"codes": found, "confidence": confidence}
-
-###############################################################################
-# REDACTION SCORING
-###############################################################################
+    return {
+        "codes": found,
+        "confidence": round(min(1.0, len(found) * 0.3), 2)
+    }
 
 def redaction_score(text: str) -> float:
     hits = sum(k in text.lower() for k in ["redact", "withheld", "b("])
     return round(min(1.0, hits * 0.25), 2)
 
 ###############################################################################
-# SEMANTIC CLUSTERING (HF SAFE HEURISTIC)
+# TEXT UTILITIES
+###############################################################################
+
+def highlight(text: str, query: str) -> str:
+    if not query:
+        return text
+    return re.sub(
+        re.escape(query),
+        lambda m: f"<mark>{m.group(0)}</mark>",
+        text,
+        flags=re.IGNORECASE,
+    )
+
+###############################################################################
+# SEMANTIC CLUSTERING + TIMELINE
 ###############################################################################
 
-def semantic_clusters(results: List[Dict]) -> Dict[str, List[Dict]]:
+def infer_year(text: str):
+    m = re.search(r"(19|20)\d{2}", text)
+    return m.group(0) if m else "Unknown"
+
+def semantic_clusters(results: List[Dict]):
     clusters = defaultdict(list)
     for r in results:
         t = r["title"].lower()
         if "uap" in t or "aerial" in t:
-            clusters["🛸 UAP / Aerial Phenomena"].append(r)
+            clusters["🛸 UAP / Aerial"].append(r)
         elif "intelligence" in t:
-            clusters["🧠 Intelligence Activities"].append(r)
+            clusters["🧠 Intelligence"].append(r)
         else:
-            clusters["📄 General Records"].append(r)
+            clusters["📄 General"].append(r)
     return clusters
 
+def timeline_view(results: List[Dict]):
+    timeline = defaultdict(list)
+    for r in results:
+        year = infer_year(r["title"] + r["url"])
+        timeline[year].append(r)
+    return dict(sorted(timeline.items(), reverse=True))
+
 ###############################################################################
-# MOCK LIVE SEARCH (REPLACE WITH REAL ADAPTERS SAFELY)
+# SEARCH
 ###############################################################################
 
-def run_search(query: str) -> List[Dict]:
-    time.sleep(0.3)
-
-    raw = [
-        {
-            "title": "UAP Task Force Preliminary Assessment",
-            "snippet": "Some material withheld under b(1) and b(3).",
-            "url": "https://www.dni.gov/files/ODNI/documents/assessments/Prelimary-Assessments-UAP-20210625.pdf",
-            "agency": "ODNI",
-            "source": "ODNI FOIA",
-        },
-        {
-            "title": "CIA Memorandum on Aerial Phenomena",
-            "snippet": "This document contains redactions under b(3).",
-            "url": "https://www.cia.gov/readingroom/docs/DOC_000001.pdf",
-            "agency": "CIA",
-            "source": "CIA FOIA",
-        },
-        {
-            "title": "Project Blue Book Summary",
-            "snippet": "Historical investigation records.",
-            "url": "https://www.archives.gov/research/military/air-force/ufos",
-            "agency": "USAF",
-            "source": "National Archives",
-        },
-    ]
-
-    allowed = []
-    for r in raw:
-        policy = AGENCY_POLICY.get(r["agency"], {})
+def run_search(query: str, semantic_refine: bool):
+    adapters = get_all_adapters()
+    results = []
+
+    for name, adapter in adapters.items():
+        policy = AGENCY_POLICY.get(name, {})
         if not policy.get("robots", False):
             continue
-        if not KILL.enabled(r["agency"]):
+        if not KILL.enabled(name):
             continue
-        allowed.append(r)
+        try:
+            res = adapter.sync_search(query)
+            results.extend(res)
+        except Exception as e:
+            KILL.disable(name, str(e))
 
-    return allowed
+    if semantic_refine and results:
+        SEMANTIC.build([r["title"] + " " + r["snippet"] for r in results])
+        ranked = SEMANTIC.search(query, k=len(results))
+        results = [r for r in results if (r["title"] + " " + r["snippet"]) in ranked]
 
-###############################################################################
-# TEXT UTILITIES
-###############################################################################
-
-def highlight(text: str, query: str) -> str:
-    if not query:
-        return text
-    return re.sub(
-        re.escape(query),
-        lambda m: f"<mark>{m.group(0)}</mark>",
-        text,
-        flags=re.IGNORECASE,
-    )
+    return results
 
 ###############################################################################
 # RENDERERS
 ###############################################################################
 
-def render_results(results: List[Dict], query: str) -> str:
-    clusters = semantic_clusters(results)
+def render_results(results, query, view):
     blocks = []
 
-    for name, items in clusters.items():
-        section = [f"## {name}"]
-        for idx, r in enumerate(items):
-            global_index = RESULT_CACHE.index(r)
-            ex = classify_exemptions(r["snippet"])
+    if view == "Timeline":
+        groups = timeline_view(results)
+    else:
+        groups = semantic_clusters(results)
 
-            section.append(
+    for name, items in groups.items():
+        blocks.append(f"## {name}")
+        for r in items:
+            idx = RESULT_CACHE.index(r)
+            ex = classify_exemptions(r["snippet"])
+            blocks.append(
                 f"""
 **{highlight(r['title'], query)}**  
-🏛️ {r['agency']} · 📊 Redaction {redaction_score(r['snippet'])}  
-⚖️ Exemptions: `{', '.join(ex['codes']) or 'None'}` (conf {ex['confidence']})  
+🏛️ {r['agency']} · 🛡️ Redaction {redaction_score(r['snippet'])}  
+⚖️ Exemptions: `{', '.join(ex['codes']) or 'None'}`  
 
-🔗 {r['url']}  
-➡️ **Select #{global_index}**
+➡️ Select **{idx}**  
+🔗 {r['url']}
 """
             )
-        blocks.append("\n\n".join(section))
+    return "\n\n".join(blocks)
 
-    return "\n\n---\n\n".join(blocks)
-
-def render_preview(index: int) -> str:
+def render_preview(idx: int):
     if not RESULT_CACHE:
-        return "_No document selected._"
+        return "_No document selected_"
 
-    r = RESULT_CACHE[index]
+    r = RESULT_CACHE[idx]
     ex = classify_exemptions(r["snippet"])
 
-    iframe = (
-        f'<iframe src="{r["url"]}" width="100%" height="520px" '
-        f'style="border:1px solid #444;border-radius:8px;"></iframe>'
-    )
-
     return f"""
-### 📄 Document Preview
+### 📄 Preview
 **{r['title']}**  
-🏛️ {r['agency']} · {r['source']}
+🏛️ {r['agency']}
 
-🛡️ Redaction Risk: **{redaction_score(r['snippet'])}**  
+🛡️ Redaction Risk: {redaction_score(r['snippet'])}  
 ⚖️ FOIA Exemptions: `{', '.join(ex['codes']) or 'None'}`  
-🔎 Confidence: **{ex['confidence']}**
 
-{iframe}
+<iframe src="{r['url']}" width="100%" height="520px"
+style="border:1px solid #444;border-radius:8px;"></iframe>
 """
 
-def agency_coverage(results: List[Dict]) -> str:
+def coverage_table(results):
     counts = defaultdict(int)
     for r in results:
         counts[r["agency"]] += 1
-
     rows = ["| Agency | Docs |", "|---|---|"]
     for k, v in sorted(counts.items(), key=lambda x: -x[1]):
         rows.append(f"| {k} | {v} |")
-
     return "\n".join(rows)
 
 ###############################################################################
-# EVENT HANDLERS
+# EVENTS
 ###############################################################################
 
-def do_search(query: str):
+def do_search(query, semantic_refine, view):
     global RESULT_CACHE, CURRENT_INDEX
-    RESULT_CACHE = run_search(query)
+    log_event("search", {"query": query})
+
+    RESULT_CACHE = run_search(query, semantic_refine)
     CURRENT_INDEX = 0
 
     return (
-        render_results(RESULT_CACHE, query),
+        render_results(RESULT_CACHE, query, view),
         render_preview(0),
-        agency_coverage(RESULT_CACHE),
-        0,
+        coverage_table(RESULT_CACHE),
+        explainability_report(RESULT_CACHE),
+        export_zip(RESULT_CACHE),
+        0
     )
 
-def select_index(idx: int):
+def select_index(idx):
     global CURRENT_INDEX
     idx = int(max(0, min(idx, len(RESULT_CACHE) - 1)))
     CURRENT_INDEX = idx
     return render_preview(idx)
 
-def next_doc():
-    global CURRENT_INDEX
-    if CURRENT_INDEX < len(RESULT_CACHE) - 1:
-        CURRENT_INDEX += 1
-    return CURRENT_INDEX, render_preview(CURRENT_INDEX)
-
-def prev_doc():
-    global CURRENT_INDEX
-    if CURRENT_INDEX > 0:
-        CURRENT_INDEX -= 1
-    return CURRENT_INDEX, render_preview(CURRENT_INDEX)
-
 ###############################################################################
 # UI
 ###############################################################################
@@ -244,7 +212,10 @@ def prev_doc():
 with gr.Blocks(theme=gr.themes.Soft()) as app:
     gr.Markdown("# 🏛️ Federated FOIA Intelligence Search")
 
-    query = gr.Textbox(label="Search public FOIA reading rooms")
+    query = gr.Textbox(label="Search public FOIA records")
+    semantic_toggle = gr.Checkbox(label="🧠 Semantic refine (FAISS)", value=False)
+    view = gr.Radio(["Clustered", "Timeline"], value="Clustered")
+
     search_btn = gr.Button("🔍 Search")
 
     with gr.Row():
@@ -255,32 +226,20 @@ with gr.Blocks(theme=gr.themes.Soft()) as app:
 
         with gr.Column(scale=7):
             preview_md = gr.Markdown()
-            with gr.Row():
-                prev_btn = gr.Button("⬅️ Prev")
-                next_btn = gr.Button("➡️ Next")
-            index_box = gr.Number(label="Selected index", precision=0)
+            index_box = gr.Number(label="Select index", precision=0)
 
-    search_btn.click(
-        do_search,
-        inputs=query,
-        outputs=[results_md, preview_md, coverage_md, index_box],
-    )
+    with gr.Accordion("🧾 Explainability Report", open=False):
+        explain_md = gr.Markdown()
 
-    index_box.change(
-        select_index,
-        inputs=index_box,
-        outputs=preview_md,
-    )
+    zip_file = gr.File(label="📦 Journalist ZIP")
 
-    next_btn.click(
-        next_doc,
-        outputs=[index_box, preview_md],
+    search_btn.click(
+        do_search,
+        inputs=[query, semantic_toggle, view],
+        outputs=[results_md, preview_md, coverage_md, explain_md, zip_file, index_box],
     )
 
-    prev_btn.click(
-        prev_doc,
-        outputs=[index_box, preview_md],
-    )
+    index_box.change(select_index, inputs=index_box, outputs=preview_md)
 
 if __name__ == "__main__":
     app.launch()