import os
import time
import json
import hashlib
import requests
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
import streamlit as st
import extra_streamlit_components as stx
from datetime import datetime, timedelta
RAG_API_URL = os.getenv("RAG_API_URL", "").rstrip("/")
RAG_API_KEY = os.getenv("RAG_API_KEY", "")
APP_USERNAME = os.getenv("USERNAME", "admin")
APP_PASSWORD = os.getenv("PASSWORD", "")
SESSION_DURATION_DAYS = 30
COOKIE_NAME = "nexus_auth_token"
st.set_page_config(
page_title="Nexus RAG",
page_icon="⬡",
layout="wide",
initial_sidebar_state="expanded",
)
st.markdown("""
""", unsafe_allow_html=True)
LUCIDE_ICONS = {
"lock": """{LUCIDE_ICONS['lock']}
NEXUS RAG
Restricted Access — Authenticate to Continue
""", unsafe_allow_html=True)
if st.session_state.get("login_error"):
st.markdown(f'{st.session_state.login_error}
', unsafe_allow_html=True)
with st.form("login_form", clear_on_submit=False):
username = st.text_input("Username", placeholder="Enter username")
password = st.text_input("Password", type="password", placeholder="Enter password")
submitted = st.form_submit_button("Sign In", use_container_width=True)
if submitted:
if not username or not password:
st.session_state.login_error = "Username and password are required."
st.rerun()
else:
success = do_login(username, password)
if success:
st.session_state.login_error = ""
st.session_state.authenticated = True
st.rerun()
else:
st.session_state.login_error = "Invalid credentials. Access denied."
st.rerun()
st.markdown("""
NEXUS RAG ENGINE — SECURED
""", unsafe_allow_html=True)
def render_sidebar(health: dict) -> str:
with st.sidebar:
status = health.get("status", "offline")
status_color = "#10b981" if status == "healthy" else "#f59e0b" if status == "degraded" else "#e94560"
status_label = status.upper() if status else "OFFLINE"
st.markdown(f"""
NEXUS RAG
Engine Dashboard
{status_label}
""", unsafe_allow_html=True)
if health:
uptime = health.get("uptime_seconds", 0)
hours = int(uptime // 3600)
minutes = int((uptime % 3600) // 60)
qdrant_ok = health.get("qdrant_connection", "error") == "ok"
cache_n = health.get("cache_entries", 0)
st.markdown(f"""
""", unsafe_allow_html=True)
pages = ["Overview", "Upload", "Test Search", "Documents", "Analytics"]
for label in pages:
is_active = st.session_state.get("current_page", "Overview") == label
if st.button(
label,
key=f"nav_{label}",
use_container_width=True,
type="primary" if is_active else "secondary",
):
st.session_state.current_page = label
st.rerun()
st.markdown("
", unsafe_allow_html=True)
st.markdown(
f""
f"Signed in as {APP_USERNAME}
",
unsafe_allow_html=True,
)
st.markdown('', unsafe_allow_html=True)
if st.button("Sign Out", key="logout_btn", use_container_width=True):
do_logout()
st.markdown("
", unsafe_allow_html=True)
return st.session_state.get("current_page", "Overview")
def page_overview(stats: dict, health: dict):
st.markdown('Overview
', unsafe_allow_html=True)
st.markdown('System status and key metrics at a glance.
', unsafe_allow_html=True)
storage = stats.get("storage", {})
documents = stats.get("documents", {})
queries = stats.get("queries", {})
c1, c2, c3, c4 = st.columns(4)
for col, val, label, sub in [
(c1, documents.get("total_parents", 0), "Parent Chunks", "Indexed parent segments"),
(c2, documents.get("total_children", 0), "Child Chunks", "Searchable vectors"),
(c3, queries.get("today", 0), "Queries Today", "RAG calls this session"),
(c4, queries.get("cache_size", 0), "Cache Entries", "In-memory cached results"),
]:
with col:
st.markdown(f"""
""", unsafe_allow_html=True)
st.markdown("Top Queries
', unsafe_allow_html=True)
if top_queries:
max_count = top_queries[0].get("count", 1)
for q in top_queries[:6]:
qtext = q.get("query", "")[:45]
count = q.get("count", 0)
pct_bar = int((count / max_count) * 100)
st.markdown(f"""
""", unsafe_allow_html=True)
else:
st.markdown('No query logs yet.
', unsafe_allow_html=True)
rate_limits = stats.get("rate_limits", {})
if rate_limits:
st.markdown('Rate Limit — Current Hour
', unsafe_allow_html=True)
rl_cols = st.columns(min(len(rate_limits), 4))
for i, (key, count) in enumerate(rate_limits.items()):
pct_rl = (count / 3000) * 100
rl_color = COLOR_SUCCESS if pct_rl < 70 else COLOR_WARNING if pct_rl < 90 else COLOR_PRIMARY
with rl_cols[i % 4]:
st.markdown(f"""
""", unsafe_allow_html=True)
def page_upload():
st.markdown('Upload Documents
', unsafe_allow_html=True)
st.markdown('Add .md or .txt files to the Nexus RAG knowledge base.
', unsafe_allow_html=True)
ok, collections_data = api_get("/collections")
existing_collections = collections_data.get("collections", []) if ok else []
with st.form("upload_form", clear_on_submit=True):
uploaded_files = st.file_uploader("Select files", type=["md", "txt"], accept_multiple_files=True, help="Max 50MB per file")
c1, c2 = st.columns(2)
with c1:
col_mode = st.selectbox("Collection mode", ["Use existing", "Create new"])
with c2:
if col_mode == "Use existing" and existing_collections:
collection_name = st.selectbox("Collection", existing_collections)
else:
collection_name = st.text_input("Collection name", placeholder="e.g. devops, tutorials")
metadata_raw = st.text_area("Metadata JSON (optional)", placeholder='{"category": "devops"}', height=70)
submitted = st.form_submit_button("Upload", use_container_width=True)
if submitted:
if not uploaded_files:
st.markdown('Select at least one file.
', unsafe_allow_html=True)
return
if not collection_name or not collection_name.strip():
st.markdown('Collection name is required.
', unsafe_allow_html=True)
return
meta_dict = {}
if metadata_raw.strip():
try:
meta_dict = json.loads(metadata_raw.strip())
except json.JSONDecodeError:
st.markdown('Invalid JSON in metadata.
', unsafe_allow_html=True)
return
progress = st.progress(0, text="Preparing...")
results_box = st.container()
success_n, fail_n = 0, 0
for i, f in enumerate(uploaded_files):
progress.progress(i / len(uploaded_files), text=f"Uploading {f.name}...")
raw = f.read()
size_mb = len(raw) / (1024 * 1024)
if size_mb > 50:
with results_box:
st.markdown(f'{f.name} exceeds 50MB ({size_mb:.1f}MB)
', unsafe_allow_html=True)
fail_n += 1
continue
ok, resp = api_post("/upload", files={"file": (f.name, raw, "text/plain")}, data={"collection": collection_name.strip(), "metadata": json.dumps(meta_dict)})
with results_box:
if ok and resp.get("success"):
st.markdown(f'{f.name} — {resp.get("parents_created",0)} parents, {resp.get("children_created",0)} children — {size_mb:.2f}MB — {resp.get("processing_time_ms",0):.0f}ms
', unsafe_allow_html=True)
success_n += 1
else:
st.markdown(f'{f.name} — {resp.get("error") or resp.get("detail","Unknown error")}
', unsafe_allow_html=True)
fail_n += 1
progress.progress(1.0, text="Done")
st.markdown("Test Search
', unsafe_allow_html=True)
st.markdown('Run live queries against the Nexus RAG engine.
', unsafe_allow_html=True)
ok, collections_data = api_get("/collections")
collections = ["all"] + (collections_data.get("collections", []) if ok else [])
with st.form("search_form"):
query_text = st.text_area("Query", placeholder="Type your query here...", height=90)
c1, c2, c3 = st.columns(3)
with c1:
selected_col = st.selectbox("Collection", collections)
with c2:
top_k = st.slider("Top K", 1, 20, 5)
with c3:
use_rerank = st.checkbox("Reranking", value=True)
submitted = st.form_submit_button("Search", use_container_width=True)
if submitted:
if not query_text.strip():
st.markdown('Enter a query.
', unsafe_allow_html=True)
return
with st.spinner("Processing..."):
t0 = time.time()
ok, resp = api_post("/query", json_data={"query": query_text.strip(), "collection": selected_col, "top_k": top_k, "use_reranking": use_rerank})
elapsed = (time.time() - t0) * 1000
if not ok:
st.markdown(f'Query failed: {resp.get("error","Unknown")}
', unsafe_allow_html=True)
return
c1, c2, c3, c4 = st.columns(4)
c1.metric("Latency", f"{resp.get('processing_time_ms', elapsed):.0f}ms")
c2.metric("Sources", len(resp.get("sources", [])))
c3.metric("Cached", "Yes" if resp.get("cached") else "No")
c4.metric("Reranked", "Yes" if use_rerank else "No")
st.markdown("Context length: {len(context):,} chars
', unsafe_allow_html=True)
st.text_area("", value=context, height=400, label_visibility="collapsed")
else:
st.markdown('No context returned.
', unsafe_allow_html=True)
with tab_src:
sources = resp.get("sources", [])
if sources:
for i, src in enumerate(sources, 1):
score = src.get("score", 0)
score_cls = "score-high" if score > 0.7 else "score-med" if score > 0.5 else "score-low"
preview = src.get("text", "")[:300]
dots = "..." if len(src.get("text", "")) > 300 else ""
st.markdown(f"""
Result #{i}
{score:.4f}
{preview}{dots}
{src.get('filename','unknown')} · {src.get('collection','unknown')} · {src.get('doc_id','')[:12]}...
""", unsafe_allow_html=True)
else:
st.markdown('No sources found.
', unsafe_allow_html=True)
def page_documents():
st.markdown('Documents
', unsafe_allow_html=True)
st.markdown('Manage all indexed documents in the knowledge base.
', unsafe_allow_html=True)
ok, collections_data = api_get("/collections")
collections = ["all"] + (collections_data.get("collections", []) if ok else [])
c1, c2 = st.columns([3, 1])
with c1:
filter_col = st.selectbox("Filter by collection", collections, key="doc_filter_col")
with c2:
st.markdown("Failed to fetch documents.
', unsafe_allow_html=True)
return
documents = docs_data.get("documents", [])
total = docs_data.get("total", 0)
st.markdown(f'{total} document(s) found
', unsafe_allow_html=True)
if not documents:
st.markdown('No documents indexed yet.
', unsafe_allow_html=True)
return
if "del_confirm" not in st.session_state:
st.session_state.del_confirm = {}
st.markdown('', unsafe_allow_html=True)
st.markdown("""
""", unsafe_allow_html=True)
for doc in documents:
doc_id = doc.get("doc_id", "")
filename = doc.get("filename", "unknown")
collection = doc.get("collection", "general")
parent_count = doc.get("parent_count", 0)
ext = "MD" if filename.endswith(".md") else "TXT"
badge_color = "badge-blue" if ext == "MD" else "badge-gray"
c1, c2, c3, c4, c5 = st.columns([3, 2, 2, 1, 1])
with c1:
st.markdown(f'
{ext} {filename} ', unsafe_allow_html=True)
with c2:
st.markdown(f'
{collection} ', unsafe_allow_html=True)
with c3:
st.markdown(f'
{doc_id[:16]}...', unsafe_allow_html=True)
with c4:
st.markdown(f'
{parent_count}
', unsafe_allow_html=True)
with c5:
if st.session_state.del_confirm.get(doc_id):
if st.button("Confirm", key=f"confirm_{doc_id}", use_container_width=True):
ok, _ = api_delete(f"/delete/{doc_id}")
if ok:
st.session_state.del_confirm[doc_id] = False
st.rerun()
else:
st.markdown('
Delete failed.
', unsafe_allow_html=True)
st.session_state.del_confirm[doc_id] = False
else:
if st.button("Delete", key=f"del_{doc_id}", use_container_width=True):
st.session_state.del_confirm[doc_id] = True
st.rerun()
st.markdown("
", unsafe_allow_html=True)
st.markdown("
Analytics
', unsafe_allow_html=True)
st.markdown('Performance insights, usage trends, and system health reports.
', unsafe_allow_html=True)
queries_data = stats.get("queries", {})
top_queries = stats.get("top_queries", [])
storage = stats.get("storage", {})
documents = stats.get("documents", {})
c1, c2, c3, c4, c5 = st.columns(5)
for col, val, label in [
(c1, queries_data.get("total_logged", 0), "Total Logged"),
(c2, queries_data.get("today", 0), "Today"),
(c3, queries_data.get("cache_size", 0), "Cache Size"),
(c4, f"{storage.get('used_mb', 0):.1f}", "Storage MB"),
(c5, f"{storage.get('percentage', 0):.1f}%", "Usage"),
]:
with col:
st.markdown(f"""
""", unsafe_allow_html=True)
st.markdown("Top Queries — Frequency
', unsafe_allow_html=True)
if top_queries:
df_q = pd.DataFrame(top_queries)
df_q.columns = ["Query", "Count"]
df_q["Query"] = df_q["Query"].str[:35]
fig = px.bar(df_q.head(10), x="Count", y="Query", orientation="h", color="Count", color_continuous_scale=["#1e1e2e", COLOR_PRIMARY])
fig.update_layout(**PLOTLY_LAYOUT, height=320, coloraxis_showscale=False)
fig.update_traces(marker_line_width=0)
st.plotly_chart(fig, use_container_width=True)
else:
st.markdown('No query data yet.
', unsafe_allow_html=True)
with row1_r:
st.markdown('Storage Distribution
', unsafe_allow_html=True)
child_count = documents.get("total_children", 0)
parent_count = documents.get("total_parents", 0)
used_mb = storage.get("used_mb", 0)
free_mb = max(storage.get("total_mb", 1024) - used_mb, 0)
if child_count + parent_count > 0:
fig_pie = go.Figure(go.Pie(
labels=["Child Vectors", "Parent Texts", "Free"],
values=[child_count * 3, parent_count * 1, max(free_mb, 1)],
hole=0.55,
marker=dict(colors=[COLOR_PRIMARY, COLOR_SECONDARY, "#1e1e2e"]),
textfont=dict(color="#6b7280", size=11),
))
fig_pie.update_layout(**PLOTLY_LAYOUT, height=320)
fig_pie.update_traces(textposition="outside")
st.plotly_chart(fig_pie, use_container_width=True)
else:
st.markdown('No storage data yet.
', unsafe_allow_html=True)
row2_l, row2_r = st.columns(2)
with row2_l:
st.markdown('Chunk Ratio Analysis
', unsafe_allow_html=True)
if parent_count > 0:
ratio = child_count / parent_count
fig_bar = go.Figure()
fig_bar.add_trace(go.Bar(
x=["Parent Chunks", "Child Chunks"],
y=[parent_count, child_count],
marker_color=[COLOR_SECONDARY, COLOR_PRIMARY],
marker_line_width=0,
text=[f"{parent_count:,}", f"{child_count:,}"],
textposition="outside",
textfont=dict(color="#6b7280", size=11),
))
fig_bar.update_layout(**PLOTLY_LAYOUT, height=280, showlegend=False)
st.plotly_chart(fig_bar, use_container_width=True)
st.markdown(f"""
Avg children per parent:
{ratio:.1f}x
""", unsafe_allow_html=True)
else:
st.markdown('No chunk data yet.
', unsafe_allow_html=True)
with row2_r:
st.markdown('Cache vs Live Queries
', unsafe_allow_html=True)
total_logged = queries_data.get("total_logged", 0)
cache_size = queries_data.get("cache_size", 0)
live_queries = max(total_logged - cache_size, 0)
if total_logged > 0:
fig_donut = go.Figure(go.Pie(
labels=["Live (Computed)", "Cached (Fast)"],
values=[live_queries, cache_size],
hole=0.6,
marker=dict(colors=[COLOR_PRIMARY, COLOR_SUCCESS]),
textfont=dict(color="#6b7280", size=11),
))
fig_donut.update_layout(**PLOTLY_LAYOUT, height=280)
fig_donut.update_traces(textposition="outside")
st.plotly_chart(fig_donut, use_container_width=True)
hit_rate = (cache_size / total_logged * 100) if total_logged > 0 else 0
st.markdown(f"""
Cache hit rate:
{hit_rate:.1f}%
""", unsafe_allow_html=True)
else:
st.markdown('No query data yet.
', unsafe_allow_html=True)
st.markdown("Storage Utilization
', unsafe_allow_html=True)
pct = storage.get("percentage", 0)
fig_bullet = go.Figure(go.Indicator(
mode="number+gauge+delta",
value=pct,
delta={"reference": 70, "increasing": {"color": COLOR_PRIMARY}, "decreasing": {"color": COLOR_SUCCESS}},
number={"suffix": "%", "font": {"color": COLOR_PRIMARY, "size": 28}},
gauge={
"shape": "bullet",
"axis": {"range": [0, 100], "tickfont": {"color": "#4a4a6a"}},
"threshold": {"line": {"color": COLOR_PRIMARY, "width": 2}, "thickness": 0.75, "value": 90},
"bgcolor": "#0a0a0f",
"steps": [
{"range": [0, 60], "color": "#0d1117"},
{"range": [60, 80], "color": "#111827"},
{"range": [80, 100], "color": "#1a0a0a"},
],
"bar": {"color": COLOR_PRIMARY},
},
title={"text": f"Used {storage.get('used_mb',0):.1f} MB of {storage.get('total_mb',1024)} MB", "font": {"color": "#4a4a6a", "size": 12}},
))
fig_bullet.update_layout(**PLOTLY_LAYOUT, height=160)
st.plotly_chart(fig_bullet, use_container_width=True)
st.markdown("Engine Configuration
', unsafe_allow_html=True)
config_items = [
("Embedding Model", "LazarusNLP/all-indobert-base-v2"),
("Reranking Model", "cross-encoder/ms-marco-MiniLM-L-6-v2"),
("Vector Dimension", "768"),
("Parent Chunk", "1500 chars"),
("Child Chunk", "500 chars"),
("Child Overlap", "50 chars"),
("Rerank Candidates", "20"),
("Cache TTL", "3600s"),
("Rate Limit", "3000 req/hr"),
("Max File Size", "50 MB"),
]
cfg_c1, cfg_c2 = st.columns(2)
for i, (key, val) in enumerate(config_items):
col = cfg_c1 if i % 2 == 0 else cfg_c2
with col:
st.markdown(f"""
{key}
{val}
""", unsafe_allow_html=True)
def main():
if not RAG_API_URL or not RAG_API_KEY:
st.markdown('RAG_API_URL and RAG_API_KEY secrets not configured.
', unsafe_allow_html=True)
return
if not APP_PASSWORD:
st.markdown('PASSWORD secret not configured.
', unsafe_allow_html=True)
return
get_cookie_manager()
if "authenticated" not in st.session_state:
st.session_state.authenticated = False
if "login_error" not in st.session_state:
st.session_state.login_error = ""
if "current_page" not in st.session_state:
st.session_state.current_page = "Overview"
if "del_confirm" not in st.session_state:
st.session_state.del_confirm = {}
if not st.session_state.authenticated:
if check_auth():
st.session_state.authenticated = True
else:
render_login()
return
health = get_health()
ok_stats, stats = api_get("/stats")
stats = stats if ok_stats else {}
page = render_sidebar(health)
if page == "Overview":
page_overview(stats, health)
elif page == "Upload":
page_upload()
elif page == "Test Search":
page_test_search()
elif page == "Documents":
page_documents()
elif page == "Analytics":
page_analytics(stats)
if __name__ == "__main__":
main()