""" P10 ยท Observability Dashboard โ€” HuggingFace Space Full dashboard: eval scores, costs, latency, SLOs, prompt versions, alerts. gradio==5.29.0 + audioop-lts for Python 3.13 compatibility. """ import os import sys import gradio as gr sys.path.insert(0, os.path.dirname(__file__)) from src.collector import get_dashboard_data from src.alerts import detect_all_anomalies SEVERITY_EMOJI = { "critical": "๐Ÿ”ด", "warning": "๐ŸŸ ", "info": "๐Ÿ”ต", "OK": "โœ…", "ELEVATED": "๐ŸŸก", "WARNING": "๐ŸŸ ", "CRITICAL": "๐Ÿ”ด", } STATUS_COLOR = { "OK": "#22d3a0", "ELEVATED": "#f59e0b", "WARNING": "#f97316", "CRITICAL": "#ef4444", } def build_eval_panel(data: dict) -> str: history = data["eval_history"] latest = data["latest_eval"] score = latest.get("avg_composite", 0) pass_rate = latest.get("pass_rate", 0) trend = "" if len(history) >= 2: prev = history[-2]["avg_composite"] curr = history[-1]["avg_composite"] delta = curr - prev trend = f" ({'โ†‘' if delta >= 0 else 'โ†“'}{abs(delta):.2f})" lines = [ f"## ๐Ÿ“Š Eval Score Trend", f"**Latest:** {score:.2f}/10{trend} ยท Pass rate: {pass_rate}%", "", "| Run | Date | Score | Pass Rate |", "|-----|------|-------|-----------|", ] for r in history[-8:]: bar = "โ–ˆ" * int(r["avg_composite"]) + "โ–‘" * (10 - int(r["avg_composite"])) lines.append( f"| `{r['run_id'][-6:]}` " f"| {r['timestamp'][:10]} " f"| `{bar}` {r['avg_composite']:.2f} " f"| {r.get('pass_rate', 0):.0f}% |" ) return "\n".join(lines) def build_slo_panel(data: dict) -> str: slo = data["slo"] lines = ["## ๐ŸŽฏ SLO Status"] lines.append("") lines.append("| Service | SLO | Error Rate | Burn Rate | Budget Left | Status |") lines.append("|---------|-----|------------|-----------|-------------|--------|") for svc, m in slo.items(): emoji = SEVERITY_EMOJI.get(m["status"], "โ“") lines.append( f"| `{svc}` " f"| {m['slo_pct']}% " f"| {m['error_rate_pct']}% " f"| {m['burn_rate']}x " f"| {m['budget_remaining_pct']}% " f"| {emoji} {m['status']} |" ) return "\n".join(lines) def build_latency_panel(data: dict) -> str: lat = data["latency"] lines = ["## โšก Latency (p50 / p95 / p99)"] lines.append("") lines.append("| Service | p50 | p95 | p99 | SLO | Status |") lines.append("|---------|-----|-----|-----|-----|--------|") for svc, m in lat.items(): ok = "โœ… OK" if m["slo_ok"] else "โŒ BREACH" lines.append( f"| `{svc}` " f"| {m['p50_ms']}ms " f"| {m['p95_ms']}ms " f"| {m['p99_ms']}ms " f"| {m['slo_ms']}ms " f"| {ok} |" ) return "\n".join(lines) def build_cost_panel(data: dict) -> str: cost = data["cost_metrics"] daily = cost["daily"] lines = [ "## ๐Ÿ’ฐ LLM Cost Tracker (7 days)", "", f"**Total 7d:** ${cost['total_7d']:.4f} ยท " f"**Avg daily:** ${cost['avg_daily']:.4f}", "", "| Date | qwen2.5-0.5b | phi-3-mini | mistral-7b | Total |", "|------|-------------|------------|------------|-------|", ] for d in daily: lines.append( f"| {d['date']} " f"| ${d.get('qwen2.5-0.5b', 0):.4f} " f"| ${d.get('phi-3-mini', 0):.4f} " f"| ${d.get('mistral-7b', 0):.4f} " f"| **${d['total']:.4f}** |" ) lines.append("") lines.append("**By model:**") for model, cost_val in cost["by_model"].items(): lines.append(f"- `{model}`: ${cost_val:.4f}") return "\n".join(lines) def build_prompt_panel(data: dict) -> str: versions = data["prompt_versions"] lines = [ "## ๐Ÿ“ Prompt Version History", "", "| Version | Date | Project | Description | Avg Score | Requests |", "|---------|------|---------|-------------|-----------|----------|", ] for v in versions: score_bar = "โ–ˆ" * int(v["avg_score"]) + "โ–‘" * (10 - int(v["avg_score"])) lines.append( f"| `{v['version']}` " f"| {v['date'][:10]} " f"| `{v['project']}` " f"| {v['description']} " f"| `{score_bar}` {v['avg_score']} " f"| {v['requests']} |" ) return "\n".join(lines) def build_alerts_panel(alerts: list) -> str: if not alerts: return "## ๐Ÿ”” Alerts\n\nโœ… **No anomalies detected**" lines = [f"## ๐Ÿ”” Alerts ({len(alerts)} active)", ""] for a in alerts: emoji = SEVERITY_EMOJI.get(a.severity, "โ“") lines += [ f"### {emoji} `{a.severity.upper()}` โ€” {a.service}", f"**{a.message}**", f"- Value: `{a.value}` ยท Threshold: `{a.threshold}`", f"- ๐Ÿ’ก {a.recommendation}", "", ] return "\n".join(lines) def refresh_dashboard() -> tuple: """Fetch all data and build all panels.""" data = get_dashboard_data() anomalies = detect_all_anomalies(data) alert_count = len([a for a in anomalies if a.severity == "critical"]) warning_count = len([a for a in anomalies if a.severity == "warning"]) header = ( f"## ๐Ÿ–ฅ๏ธ LLM Observability Dashboard\n" f"**Last updated:** {data['timestamp'][:19]} UTC ยท " f"๐Ÿ”ด {alert_count} critical ยท ๐ŸŸ  {warning_count} warnings\n\n" f"_P10 ยท Staff SRE + AI Engineer Portfolio โ€” ties together P06, P08, P09_" ) return ( header, build_alerts_panel(anomalies), build_eval_panel(data), build_slo_panel(data), build_latency_panel(data), build_cost_panel(data), build_prompt_panel(data), ) # โ”€โ”€ Gradio UI โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ with gr.Blocks(title="P10 ยท Observability Dashboard", theme=gr.themes.Soft()) as demo: gr.Markdown(""" # ๐Ÿ–ฅ๏ธ P10 ยท LLM Observability Dashboard **Staff SRE + AI Engineer Portfolio** Unified observability for LLM systems โ€” eval scores (P09), SLO burn rates (P08 pattern), latency percentiles, cost tracking, prompt versioning, and anomaly alerts. Click **Refresh** to load live data. """) refresh_btn = gr.Button("๐Ÿ”„ Refresh Dashboard", variant="primary", size="lg") header_out = gr.Markdown() with gr.Row(): with gr.Column(): alerts_out = gr.Markdown() with gr.Column(): eval_out = gr.Markdown() with gr.Row(): with gr.Column(): slo_out = gr.Markdown() with gr.Column(): latency_out = gr.Markdown() cost_out = gr.Markdown() prompt_out = gr.Markdown() with gr.Accordion("๐Ÿ“– Architecture", open=False): gr.Markdown(""" ## How P10 connects to the rest of the portfolio | Data source | Project | What it provides | |-------------|---------|-----------------| | SQLite eval DB | P09 | Score history + regression tracking | | Mock Prometheus | P08 pattern | SLO burn rates + error budget | | Mock cost tracker | P10 | Token cost by model per day | | Mock Jaeger | P10 | Latency p50/p95/p99 per service | | Prompt store | P10 | Version history + score per version | **In production** you would replace mocks with: - Prometheus for SLO/latency - LangSmith for traces + cost - PagerDuty for alert routing - A real prompt registry (MLflow or custom) **SRE additions:** - Anomaly detection on all metrics (burn rate, cost spike, eval regression) - Alert severity: critical pages immediately, warning creates ticket - Error budget burn rate tracked and projected - Prompt version tied to eval score โ€” shows which prompt version caused regression """) gr.Markdown(""" --- [GitHub](https://github.com/amarshiv86/p10-observability) ยท [P09 Eval](https://huggingface.co/spaces/amarshiv86/p09-llm-eval) ยท [P08 Agent](https://huggingface.co/spaces/amarshiv86/p08-sre-agent) ยท [Portfolio](https://github.com/amarshiv86) """) refresh_btn.click( fn=refresh_dashboard, outputs=[header_out, alerts_out, eval_out, slo_out, latency_out, cost_out, prompt_out], ) # Auto-load on startup demo.load( fn=refresh_dashboard, outputs=[header_out, alerts_out, eval_out, slo_out, latency_out, cost_out, prompt_out], ) demo.launch()