| """ |
| P10 · Observability Dashboard — HuggingFace Space |
| Full dashboard: eval scores, costs, latency, SLOs, prompt versions, alerts. |
| gradio==5.29.0 + audioop-lts for Python 3.13 compatibility. |
| """ |
|
|
| import os |
| import sys |
|
|
| import gradio as gr |
|
|
| sys.path.insert(0, os.path.dirname(__file__)) |
| from src.collector import get_dashboard_data |
| from src.alerts import detect_all_anomalies |
|
|
| SEVERITY_EMOJI = { |
| "critical": "🔴", |
| "warning": "🟠", |
| "info": "🔵", |
| "OK": "✅", |
| "ELEVATED": "🟡", |
| "WARNING": "🟠", |
| "CRITICAL": "🔴", |
| } |
|
|
| STATUS_COLOR = { |
| "OK": "#22d3a0", |
| "ELEVATED": "#f59e0b", |
| "WARNING": "#f97316", |
| "CRITICAL": "#ef4444", |
| } |
|
|
|
|
| def build_eval_panel(data: dict) -> str: |
| history = data["eval_history"] |
| latest = data["latest_eval"] |
| score = latest.get("avg_composite", 0) |
| pass_rate = latest.get("pass_rate", 0) |
|
|
| trend = "" |
| if len(history) >= 2: |
| prev = history[-2]["avg_composite"] |
| curr = history[-1]["avg_composite"] |
| delta = curr - prev |
| trend = f" ({'↑' if delta >= 0 else '↓'}{abs(delta):.2f})" |
|
|
| lines = [ |
| f"## 📊 Eval Score Trend", |
| f"**Latest:** {score:.2f}/10{trend} · Pass rate: {pass_rate}%", |
| "", |
| "| Run | Date | Score | Pass Rate |", |
| "|-----|------|-------|-----------|", |
| ] |
| for r in history[-8:]: |
| bar = "█" * int(r["avg_composite"]) + "░" * (10 - int(r["avg_composite"])) |
| lines.append( |
| f"| `{r['run_id'][-6:]}` " |
| f"| {r['timestamp'][:10]} " |
| f"| `{bar}` {r['avg_composite']:.2f} " |
| f"| {r.get('pass_rate', 0):.0f}% |" |
| ) |
| return "\n".join(lines) |
|
|
|
|
| def build_slo_panel(data: dict) -> str: |
| slo = data["slo"] |
| lines = ["## 🎯 SLO Status"] |
| lines.append("") |
| lines.append("| Service | SLO | Error Rate | Burn Rate | Budget Left | Status |") |
| lines.append("|---------|-----|------------|-----------|-------------|--------|") |
| for svc, m in slo.items(): |
| emoji = SEVERITY_EMOJI.get(m["status"], "❓") |
| lines.append( |
| f"| `{svc}` " |
| f"| {m['slo_pct']}% " |
| f"| {m['error_rate_pct']}% " |
| f"| {m['burn_rate']}x " |
| f"| {m['budget_remaining_pct']}% " |
| f"| {emoji} {m['status']} |" |
| ) |
| return "\n".join(lines) |
|
|
|
|
| def build_latency_panel(data: dict) -> str: |
| lat = data["latency"] |
| lines = ["## ⚡ Latency (p50 / p95 / p99)"] |
| lines.append("") |
| lines.append("| Service | p50 | p95 | p99 | SLO | Status |") |
| lines.append("|---------|-----|-----|-----|-----|--------|") |
| for svc, m in lat.items(): |
| ok = "✅ OK" if m["slo_ok"] else "❌ BREACH" |
| lines.append( |
| f"| `{svc}` " |
| f"| {m['p50_ms']}ms " |
| f"| {m['p95_ms']}ms " |
| f"| {m['p99_ms']}ms " |
| f"| {m['slo_ms']}ms " |
| f"| {ok} |" |
| ) |
| return "\n".join(lines) |
|
|
|
|
| def build_cost_panel(data: dict) -> str: |
| cost = data["cost_metrics"] |
| daily = cost["daily"] |
| lines = [ |
| "## 💰 LLM Cost Tracker (7 days)", |
| "", |
| f"**Total 7d:** ${cost['total_7d']:.4f} · " |
| f"**Avg daily:** ${cost['avg_daily']:.4f}", |
| "", |
| "| Date | qwen2.5-0.5b | phi-3-mini | mistral-7b | Total |", |
| "|------|-------------|------------|------------|-------|", |
| ] |
| for d in daily: |
| lines.append( |
| f"| {d['date']} " |
| f"| ${d.get('qwen2.5-0.5b', 0):.4f} " |
| f"| ${d.get('phi-3-mini', 0):.4f} " |
| f"| ${d.get('mistral-7b', 0):.4f} " |
| f"| **${d['total']:.4f}** |" |
| ) |
| lines.append("") |
| lines.append("**By model:**") |
| for model, cost_val in cost["by_model"].items(): |
| lines.append(f"- `{model}`: ${cost_val:.4f}") |
| return "\n".join(lines) |
|
|
|
|
| def build_prompt_panel(data: dict) -> str: |
| versions = data["prompt_versions"] |
| lines = [ |
| "## 📝 Prompt Version History", |
| "", |
| "| Version | Date | Project | Description | Avg Score | Requests |", |
| "|---------|------|---------|-------------|-----------|----------|", |
| ] |
| for v in versions: |
| score_bar = "█" * int(v["avg_score"]) + "░" * (10 - int(v["avg_score"])) |
| lines.append( |
| f"| `{v['version']}` " |
| f"| {v['date'][:10]} " |
| f"| `{v['project']}` " |
| f"| {v['description']} " |
| f"| `{score_bar}` {v['avg_score']} " |
| f"| {v['requests']} |" |
| ) |
| return "\n".join(lines) |
|
|
|
|
| def build_alerts_panel(alerts: list) -> str: |
| if not alerts: |
| return "## 🔔 Alerts\n\n✅ **No anomalies detected**" |
|
|
| lines = [f"## 🔔 Alerts ({len(alerts)} active)", ""] |
| for a in alerts: |
| emoji = SEVERITY_EMOJI.get(a.severity, "❓") |
| lines += [ |
| f"### {emoji} `{a.severity.upper()}` — {a.service}", |
| f"**{a.message}**", |
| f"- Value: `{a.value}` · Threshold: `{a.threshold}`", |
| f"- 💡 {a.recommendation}", |
| "", |
| ] |
| return "\n".join(lines) |
|
|
|
|
| def refresh_dashboard() -> tuple: |
| """Fetch all data and build all panels.""" |
| data = get_dashboard_data() |
| anomalies = detect_all_anomalies(data) |
|
|
| alert_count = len([a for a in anomalies if a.severity == "critical"]) |
| warning_count = len([a for a in anomalies if a.severity == "warning"]) |
|
|
| header = ( |
| f"## 🖥️ LLM Observability Dashboard\n" |
| f"**Last updated:** {data['timestamp'][:19]} UTC · " |
| f"🔴 {alert_count} critical · 🟠 {warning_count} warnings\n\n" |
| f"_P10 · Staff SRE + AI Engineer Portfolio — ties together P06, P08, P09_" |
| ) |
|
|
| return ( |
| header, |
| build_alerts_panel(anomalies), |
| build_eval_panel(data), |
| build_slo_panel(data), |
| build_latency_panel(data), |
| build_cost_panel(data), |
| build_prompt_panel(data), |
| ) |
|
|
|
|
| |
| with gr.Blocks(title="P10 · Observability Dashboard", theme=gr.themes.Soft()) as demo: |
|
|
| gr.Markdown(""" |
| # 🖥️ P10 · LLM Observability Dashboard |
| **Staff SRE + AI Engineer Portfolio** |
| |
| Unified observability for LLM systems — eval scores (P09), SLO burn rates (P08 pattern), |
| latency percentiles, cost tracking, prompt versioning, and anomaly alerts. |
| |
| Click **Refresh** to load live data. |
| """) |
|
|
| refresh_btn = gr.Button("🔄 Refresh Dashboard", variant="primary", size="lg") |
|
|
| header_out = gr.Markdown() |
|
|
| with gr.Row(): |
| with gr.Column(): |
| alerts_out = gr.Markdown() |
| with gr.Column(): |
| eval_out = gr.Markdown() |
|
|
| with gr.Row(): |
| with gr.Column(): |
| slo_out = gr.Markdown() |
| with gr.Column(): |
| latency_out = gr.Markdown() |
|
|
| cost_out = gr.Markdown() |
| prompt_out = gr.Markdown() |
|
|
| with gr.Accordion("📖 Architecture", open=False): |
| gr.Markdown(""" |
| ## How P10 connects to the rest of the portfolio |
| |
| | Data source | Project | What it provides | |
| |-------------|---------|-----------------| |
| | SQLite eval DB | P09 | Score history + regression tracking | |
| | Mock Prometheus | P08 pattern | SLO burn rates + error budget | |
| | Mock cost tracker | P10 | Token cost by model per day | |
| | Mock Jaeger | P10 | Latency p50/p95/p99 per service | |
| | Prompt store | P10 | Version history + score per version | |
| |
| **In production** you would replace mocks with: |
| - Prometheus for SLO/latency |
| - LangSmith for traces + cost |
| - PagerDuty for alert routing |
| - A real prompt registry (MLflow or custom) |
| |
| **SRE additions:** |
| - Anomaly detection on all metrics (burn rate, cost spike, eval regression) |
| - Alert severity: critical pages immediately, warning creates ticket |
| - Error budget burn rate tracked and projected |
| - Prompt version tied to eval score — shows which prompt version caused regression |
| """) |
|
|
| gr.Markdown(""" |
| --- |
| [GitHub](https://github.com/amarshiv86/p10-observability) · |
| [P09 Eval](https://huggingface.co/spaces/amarshiv86/p09-llm-eval) · |
| [P08 Agent](https://huggingface.co/spaces/amarshiv86/p08-sre-agent) · |
| [Portfolio](https://github.com/amarshiv86) |
| """) |
|
|
| refresh_btn.click( |
| fn=refresh_dashboard, |
| outputs=[header_out, alerts_out, eval_out, slo_out, |
| latency_out, cost_out, prompt_out], |
| ) |
|
|
| |
| demo.load( |
| fn=refresh_dashboard, |
| outputs=[header_out, alerts_out, eval_out, slo_out, |
| latency_out, cost_out, prompt_out], |
| ) |
|
|
| demo.launch() |
|
|
