Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -15,11 +15,9 @@ from ingest.dod_special_extended import (
|
|
| 15 |
SpecialActivitiesExtendedAdapter
|
| 16 |
)
|
| 17 |
|
| 18 |
-
|
| 19 |
-
CIAAdapter(), # stub
|
| 20 |
-
FBIAdapter(), # live
|
| 21 |
-
]
|
| 22 |
|
|
|
|
| 23 |
EXTENDED_ADAPTERS = [
|
| 24 |
CIAExtendedAdapter(),
|
| 25 |
NSAExtendedAdapter(),
|
|
@@ -30,12 +28,8 @@ EXTENDED_ADAPTERS = [
|
|
| 30 |
SpecialActivitiesExtendedAdapter()
|
| 31 |
]
|
| 32 |
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
adapters = list(BASE_ADAPTERS)
|
| 36 |
-
if extended_mode:
|
| 37 |
-
adapters.extend(EXTENDED_ADAPTERS)
|
| 38 |
-
|
| 39 |
tasks = [a.search(query) for a in adapters]
|
| 40 |
results = await asyncio.gather(*tasks, return_exceptions=True)
|
| 41 |
|
|
@@ -45,32 +39,29 @@ async def federated_search_async(query: str, extended_mode: bool) -> List[Dict]:
|
|
| 45 |
flat.extend(r)
|
| 46 |
return flat
|
| 47 |
|
| 48 |
-
|
| 49 |
-
|
|
|
|
|
|
|
| 50 |
if r.get("extended"):
|
| 51 |
-
|
| 52 |
-
else
|
| 53 |
-
badge = "π’ LIVE" if r.get("live") else "βͺ STUB"
|
| 54 |
|
|
|
|
| 55 |
return f"""
|
| 56 |
### {r['title']}
|
| 57 |
-
**{badge} Β· {r['source']}**
|
| 58 |
|
| 59 |
{r['snippet']}
|
| 60 |
|
| 61 |
π {r['url']}
|
| 62 |
"""
|
| 63 |
|
| 64 |
-
|
| 65 |
-
def run_search(query: str, hide_stubs: bool, extended_mode: bool):
|
| 66 |
loop = asyncio.get_event_loop()
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
else:
|
| 71 |
-
results = loop.run_until_complete(
|
| 72 |
-
federated_search_async(query, extended_mode)
|
| 73 |
-
)
|
| 74 |
|
| 75 |
if hide_stubs:
|
| 76 |
results = [r for r in results if r.get("live")]
|
|
@@ -78,30 +69,44 @@ def run_search(query: str, hide_stubs: bool, extended_mode: bool):
|
|
| 78 |
if not results:
|
| 79 |
return "No results found."
|
| 80 |
|
| 81 |
-
return "\n\n---\n\n".join(
|
| 82 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
|
| 84 |
-
with gr.Blocks(
|
|
|
|
|
|
|
| 85 |
gr.Markdown("# π FOIA Federated Search")
|
| 86 |
gr.Markdown(
|
| 87 |
-
"
|
| 88 |
-
"
|
| 89 |
-
"- π’ Live = public indexed source\n"
|
| 90 |
-
"- π£ Extended = advanced public sources (opt-in)"
|
| 91 |
)
|
| 92 |
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
|
|
|
|
|
|
|
|
|
| 99 |
output = gr.Markdown()
|
|
|
|
| 100 |
|
| 101 |
-
gr.
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
|
|
|
|
|
|
| 106 |
|
| 107 |
demo.launch()
|
|
|
|
| 15 |
SpecialActivitiesExtendedAdapter
|
| 16 |
)
|
| 17 |
|
| 18 |
+
import saved_searches
|
|
|
|
|
|
|
|
|
|
| 19 |
|
| 20 |
+
BASE_ADAPTERS = [CIAAdapter(), FBIAdapter()]
|
| 21 |
EXTENDED_ADAPTERS = [
|
| 22 |
CIAExtendedAdapter(),
|
| 23 |
NSAExtendedAdapter(),
|
|
|
|
| 28 |
SpecialActivitiesExtendedAdapter()
|
| 29 |
]
|
| 30 |
|
| 31 |
+
async def federated_search_async(query: str, extended: bool):
|
| 32 |
+
adapters = BASE_ADAPTERS + (EXTENDED_ADAPTERS if extended else [])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
tasks = [a.search(query) for a in adapters]
|
| 34 |
results = await asyncio.gather(*tasks, return_exceptions=True)
|
| 35 |
|
|
|
|
| 39 |
flat.extend(r)
|
| 40 |
return flat
|
| 41 |
|
| 42 |
+
def badge(r):
|
| 43 |
+
h = r.get("health", {})
|
| 44 |
+
latency = f"{h.get('latency_ms','?')}ms"
|
| 45 |
+
status = "π’" if h.get("ok") else "π΄"
|
| 46 |
if r.get("extended"):
|
| 47 |
+
return f"π£ EXT Β· {status} {latency}"
|
| 48 |
+
return "π’ LIVE" if r.get("live") else "βͺ STUB"
|
|
|
|
| 49 |
|
| 50 |
+
def render(r):
|
| 51 |
return f"""
|
| 52 |
### {r['title']}
|
| 53 |
+
**{badge(r)} Β· {r['source']}**
|
| 54 |
|
| 55 |
{r['snippet']}
|
| 56 |
|
| 57 |
π {r['url']}
|
| 58 |
"""
|
| 59 |
|
| 60 |
+
def run_search(query, hide_stubs, extended):
|
|
|
|
| 61 |
loop = asyncio.get_event_loop()
|
| 62 |
+
results = loop.run_until_complete(federated_search_async(query, extended))
|
| 63 |
+
|
| 64 |
+
saved_searches.save(query, extended)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
|
| 66 |
if hide_stubs:
|
| 67 |
results = [r for r in results if r.get("live")]
|
|
|
|
| 69 |
if not results:
|
| 70 |
return "No results found."
|
| 71 |
|
| 72 |
+
return "\n\n---\n\n".join(render(r) for r in results)
|
| 73 |
|
| 74 |
+
def show_saved():
|
| 75 |
+
rows = saved_searches.list_saved()
|
| 76 |
+
if not rows:
|
| 77 |
+
return "No saved searches yet."
|
| 78 |
+
return "\n".join(
|
| 79 |
+
f"- **{r['query']}** (extended={r['extended']})"
|
| 80 |
+
for r in rows
|
| 81 |
+
)
|
| 82 |
|
| 83 |
+
with gr.Blocks(css="""
|
| 84 |
+
.gradio-container { max-width: 1100px }
|
| 85 |
+
""") as demo:
|
| 86 |
gr.Markdown("# π FOIA Federated Search")
|
| 87 |
gr.Markdown(
|
| 88 |
+
"Transparent federated search across public FOIA reading rooms.\n\n"
|
| 89 |
+
"**Badges**: βͺ Stub Β· π’ Live Β· π£ Extended Β· π΄ Unhealthy"
|
|
|
|
|
|
|
| 90 |
)
|
| 91 |
|
| 92 |
+
with gr.Row():
|
| 93 |
+
query = gr.Textbox(label="Search term", scale=3)
|
| 94 |
+
hide_stubs = gr.Checkbox(label="Hide stub sources", value=False)
|
| 95 |
+
extended = gr.Checkbox(
|
| 96 |
+
label="Enable Extended Features",
|
| 97 |
+
value=False,
|
| 98 |
+
info="Advanced public sources. Disabled by default."
|
| 99 |
+
)
|
| 100 |
+
|
| 101 |
output = gr.Markdown()
|
| 102 |
+
saved = gr.Markdown()
|
| 103 |
|
| 104 |
+
with gr.Row():
|
| 105 |
+
gr.Button("Search").click(
|
| 106 |
+
run_search,
|
| 107 |
+
inputs=[query, hide_stubs, extended],
|
| 108 |
+
outputs=output
|
| 109 |
+
)
|
| 110 |
+
gr.Button("Saved Searches").click(show_saved, outputs=saved)
|
| 111 |
|
| 112 |
demo.launch()
|