Upload folder using huggingface_hub
Browse files- README.md +13 -6
- app.py +272 -0
- requirements.txt +2 -0
- src/alerts.py +149 -0
- src/collector.py +234 -0
README.md
CHANGED
|
@@ -1,13 +1,20 @@
|
|
| 1 |
---
|
| 2 |
-
title: P10 Observability
|
| 3 |
-
emoji:
|
| 4 |
colorFrom: indigo
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version:
|
| 8 |
-
python_version: '3.13'
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
| 11 |
---
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: P10 LLM Observability Dashboard
|
| 3 |
+
emoji: π₯οΈ
|
| 4 |
colorFrom: indigo
|
| 5 |
+
colorTo: purple
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 5.29.0
|
|
|
|
| 8 |
app_file: app.py
|
| 9 |
pinned: false
|
| 10 |
---
|
| 11 |
|
| 12 |
+
# P10 Β· LLM Observability Dashboard
|
| 13 |
+
|
| 14 |
+
Unified observability for LLM systems β eval scores, SLO burn rates,
|
| 15 |
+
latency percentiles, cost tracking, prompt versioning, and anomaly alerts.
|
| 16 |
+
|
| 17 |
+
Ties together P06 (code review), P08 (SRE agent), P09 (eval framework).
|
| 18 |
+
Part of the [Staff SRE Β· AI Engineer Portfolio](https://github.com/amarshiv86).
|
| 19 |
+
|
| 20 |
+
> Auto-loads on startup. Click Refresh to update.
|
app.py
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
P10 Β· Observability Dashboard β HuggingFace Space
|
| 3 |
+
Full dashboard: eval scores, costs, latency, SLOs, prompt versions, alerts.
|
| 4 |
+
gradio==5.29.0 + audioop-lts for Python 3.13 compatibility.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import os
|
| 8 |
+
import sys
|
| 9 |
+
|
| 10 |
+
import gradio as gr
|
| 11 |
+
|
| 12 |
+
sys.path.insert(0, os.path.dirname(__file__))
|
| 13 |
+
from src.collector import get_dashboard_data
|
| 14 |
+
from src.alerts import detect_all_anomalies
|
| 15 |
+
|
| 16 |
+
SEVERITY_EMOJI = {
|
| 17 |
+
"critical": "π΄",
|
| 18 |
+
"warning": "π ",
|
| 19 |
+
"info": "π΅",
|
| 20 |
+
"OK": "β
",
|
| 21 |
+
"ELEVATED": "π‘",
|
| 22 |
+
"WARNING": "π ",
|
| 23 |
+
"CRITICAL": "π΄",
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
STATUS_COLOR = {
|
| 27 |
+
"OK": "#22d3a0",
|
| 28 |
+
"ELEVATED": "#f59e0b",
|
| 29 |
+
"WARNING": "#f97316",
|
| 30 |
+
"CRITICAL": "#ef4444",
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def build_eval_panel(data: dict) -> str:
|
| 35 |
+
history = data["eval_history"]
|
| 36 |
+
latest = data["latest_eval"]
|
| 37 |
+
score = latest.get("avg_composite", 0)
|
| 38 |
+
pass_rate = latest.get("pass_rate", 0)
|
| 39 |
+
|
| 40 |
+
trend = ""
|
| 41 |
+
if len(history) >= 2:
|
| 42 |
+
prev = history[-2]["avg_composite"]
|
| 43 |
+
curr = history[-1]["avg_composite"]
|
| 44 |
+
delta = curr - prev
|
| 45 |
+
trend = f" ({'β' if delta >= 0 else 'β'}{abs(delta):.2f})"
|
| 46 |
+
|
| 47 |
+
lines = [
|
| 48 |
+
f"## π Eval Score Trend",
|
| 49 |
+
f"**Latest:** {score:.2f}/10{trend} Β· Pass rate: {pass_rate}%",
|
| 50 |
+
"",
|
| 51 |
+
"| Run | Date | Score | Pass Rate |",
|
| 52 |
+
"|-----|------|-------|-----------|",
|
| 53 |
+
]
|
| 54 |
+
for r in history[-8:]:
|
| 55 |
+
bar = "β" * int(r["avg_composite"]) + "β" * (10 - int(r["avg_composite"]))
|
| 56 |
+
lines.append(
|
| 57 |
+
f"| `{r['run_id'][-6:]}` "
|
| 58 |
+
f"| {r['timestamp'][:10]} "
|
| 59 |
+
f"| `{bar}` {r['avg_composite']:.2f} "
|
| 60 |
+
f"| {r.get('pass_rate', 0):.0f}% |"
|
| 61 |
+
)
|
| 62 |
+
return "\n".join(lines)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def build_slo_panel(data: dict) -> str:
|
| 66 |
+
slo = data["slo"]
|
| 67 |
+
lines = ["## π― SLO Status"]
|
| 68 |
+
lines.append("")
|
| 69 |
+
lines.append("| Service | SLO | Error Rate | Burn Rate | Budget Left | Status |")
|
| 70 |
+
lines.append("|---------|-----|------------|-----------|-------------|--------|")
|
| 71 |
+
for svc, m in slo.items():
|
| 72 |
+
emoji = SEVERITY_EMOJI.get(m["status"], "β")
|
| 73 |
+
lines.append(
|
| 74 |
+
f"| `{svc}` "
|
| 75 |
+
f"| {m['slo_pct']}% "
|
| 76 |
+
f"| {m['error_rate_pct']}% "
|
| 77 |
+
f"| {m['burn_rate']}x "
|
| 78 |
+
f"| {m['budget_remaining_pct']}% "
|
| 79 |
+
f"| {emoji} {m['status']} |"
|
| 80 |
+
)
|
| 81 |
+
return "\n".join(lines)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def build_latency_panel(data: dict) -> str:
|
| 85 |
+
lat = data["latency"]
|
| 86 |
+
lines = ["## β‘ Latency (p50 / p95 / p99)"]
|
| 87 |
+
lines.append("")
|
| 88 |
+
lines.append("| Service | p50 | p95 | p99 | SLO | Status |")
|
| 89 |
+
lines.append("|---------|-----|-----|-----|-----|--------|")
|
| 90 |
+
for svc, m in lat.items():
|
| 91 |
+
ok = "β
OK" if m["slo_ok"] else "β BREACH"
|
| 92 |
+
lines.append(
|
| 93 |
+
f"| `{svc}` "
|
| 94 |
+
f"| {m['p50_ms']}ms "
|
| 95 |
+
f"| {m['p95_ms']}ms "
|
| 96 |
+
f"| {m['p99_ms']}ms "
|
| 97 |
+
f"| {m['slo_ms']}ms "
|
| 98 |
+
f"| {ok} |"
|
| 99 |
+
)
|
| 100 |
+
return "\n".join(lines)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def build_cost_panel(data: dict) -> str:
|
| 104 |
+
cost = data["cost_metrics"]
|
| 105 |
+
daily = cost["daily"]
|
| 106 |
+
lines = [
|
| 107 |
+
"## π° LLM Cost Tracker (7 days)",
|
| 108 |
+
"",
|
| 109 |
+
f"**Total 7d:** ${cost['total_7d']:.4f} Β· "
|
| 110 |
+
f"**Avg daily:** ${cost['avg_daily']:.4f}",
|
| 111 |
+
"",
|
| 112 |
+
"| Date | qwen2.5-0.5b | phi-3-mini | mistral-7b | Total |",
|
| 113 |
+
"|------|-------------|------------|------------|-------|",
|
| 114 |
+
]
|
| 115 |
+
for d in daily:
|
| 116 |
+
lines.append(
|
| 117 |
+
f"| {d['date']} "
|
| 118 |
+
f"| ${d.get('qwen2.5-0.5b', 0):.4f} "
|
| 119 |
+
f"| ${d.get('phi-3-mini', 0):.4f} "
|
| 120 |
+
f"| ${d.get('mistral-7b', 0):.4f} "
|
| 121 |
+
f"| **${d['total']:.4f}** |"
|
| 122 |
+
)
|
| 123 |
+
lines.append("")
|
| 124 |
+
lines.append("**By model:**")
|
| 125 |
+
for model, cost_val in cost["by_model"].items():
|
| 126 |
+
lines.append(f"- `{model}`: ${cost_val:.4f}")
|
| 127 |
+
return "\n".join(lines)
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def build_prompt_panel(data: dict) -> str:
|
| 131 |
+
versions = data["prompt_versions"]
|
| 132 |
+
lines = [
|
| 133 |
+
"## π Prompt Version History",
|
| 134 |
+
"",
|
| 135 |
+
"| Version | Date | Project | Description | Avg Score | Requests |",
|
| 136 |
+
"|---------|------|---------|-------------|-----------|----------|",
|
| 137 |
+
]
|
| 138 |
+
for v in versions:
|
| 139 |
+
score_bar = "β" * int(v["avg_score"]) + "β" * (10 - int(v["avg_score"]))
|
| 140 |
+
lines.append(
|
| 141 |
+
f"| `{v['version']}` "
|
| 142 |
+
f"| {v['date'][:10]} "
|
| 143 |
+
f"| `{v['project']}` "
|
| 144 |
+
f"| {v['description']} "
|
| 145 |
+
f"| `{score_bar}` {v['avg_score']} "
|
| 146 |
+
f"| {v['requests']} |"
|
| 147 |
+
)
|
| 148 |
+
return "\n".join(lines)
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def build_alerts_panel(alerts: list) -> str:
|
| 152 |
+
if not alerts:
|
| 153 |
+
return "## π Alerts\n\nβ
**No anomalies detected**"
|
| 154 |
+
|
| 155 |
+
lines = [f"## π Alerts ({len(alerts)} active)", ""]
|
| 156 |
+
for a in alerts:
|
| 157 |
+
emoji = SEVERITY_EMOJI.get(a.severity, "β")
|
| 158 |
+
lines += [
|
| 159 |
+
f"### {emoji} `{a.severity.upper()}` β {a.service}",
|
| 160 |
+
f"**{a.message}**",
|
| 161 |
+
f"- Value: `{a.value}` Β· Threshold: `{a.threshold}`",
|
| 162 |
+
f"- π‘ {a.recommendation}",
|
| 163 |
+
"",
|
| 164 |
+
]
|
| 165 |
+
return "\n".join(lines)
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def refresh_dashboard() -> tuple:
|
| 169 |
+
"""Fetch all data and build all panels."""
|
| 170 |
+
data = get_dashboard_data()
|
| 171 |
+
anomalies = detect_all_anomalies(data)
|
| 172 |
+
|
| 173 |
+
alert_count = len([a for a in anomalies if a.severity == "critical"])
|
| 174 |
+
warning_count = len([a for a in anomalies if a.severity == "warning"])
|
| 175 |
+
|
| 176 |
+
header = (
|
| 177 |
+
f"## π₯οΈ LLM Observability Dashboard\n"
|
| 178 |
+
f"**Last updated:** {data['timestamp'][:19]} UTC Β· "
|
| 179 |
+
f"π΄ {alert_count} critical Β· π {warning_count} warnings\n\n"
|
| 180 |
+
f"_P10 Β· Staff SRE + AI Engineer Portfolio β ties together P06, P08, P09_"
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
return (
|
| 184 |
+
header,
|
| 185 |
+
build_alerts_panel(anomalies),
|
| 186 |
+
build_eval_panel(data),
|
| 187 |
+
build_slo_panel(data),
|
| 188 |
+
build_latency_panel(data),
|
| 189 |
+
build_cost_panel(data),
|
| 190 |
+
build_prompt_panel(data),
|
| 191 |
+
)
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
# ββ Gradio UI ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 195 |
+
with gr.Blocks(title="P10 Β· Observability Dashboard", theme=gr.themes.Soft()) as demo:
|
| 196 |
+
|
| 197 |
+
gr.Markdown("""
|
| 198 |
+
# π₯οΈ P10 Β· LLM Observability Dashboard
|
| 199 |
+
**Staff SRE + AI Engineer Portfolio**
|
| 200 |
+
|
| 201 |
+
Unified observability for LLM systems β eval scores (P09), SLO burn rates (P08 pattern),
|
| 202 |
+
latency percentiles, cost tracking, prompt versioning, and anomaly alerts.
|
| 203 |
+
|
| 204 |
+
Click **Refresh** to load live data.
|
| 205 |
+
""")
|
| 206 |
+
|
| 207 |
+
refresh_btn = gr.Button("π Refresh Dashboard", variant="primary", size="lg")
|
| 208 |
+
|
| 209 |
+
header_out = gr.Markdown()
|
| 210 |
+
|
| 211 |
+
with gr.Row():
|
| 212 |
+
with gr.Column():
|
| 213 |
+
alerts_out = gr.Markdown()
|
| 214 |
+
with gr.Column():
|
| 215 |
+
eval_out = gr.Markdown()
|
| 216 |
+
|
| 217 |
+
with gr.Row():
|
| 218 |
+
with gr.Column():
|
| 219 |
+
slo_out = gr.Markdown()
|
| 220 |
+
with gr.Column():
|
| 221 |
+
latency_out = gr.Markdown()
|
| 222 |
+
|
| 223 |
+
cost_out = gr.Markdown()
|
| 224 |
+
prompt_out = gr.Markdown()
|
| 225 |
+
|
| 226 |
+
with gr.Accordion("π Architecture", open=False):
|
| 227 |
+
gr.Markdown("""
|
| 228 |
+
## How P10 connects to the rest of the portfolio
|
| 229 |
+
|
| 230 |
+
| Data source | Project | What it provides |
|
| 231 |
+
|-------------|---------|-----------------|
|
| 232 |
+
| SQLite eval DB | P09 | Score history + regression tracking |
|
| 233 |
+
| Mock Prometheus | P08 pattern | SLO burn rates + error budget |
|
| 234 |
+
| Mock cost tracker | P10 | Token cost by model per day |
|
| 235 |
+
| Mock Jaeger | P10 | Latency p50/p95/p99 per service |
|
| 236 |
+
| Prompt store | P10 | Version history + score per version |
|
| 237 |
+
|
| 238 |
+
**In production** you would replace mocks with:
|
| 239 |
+
- Prometheus for SLO/latency
|
| 240 |
+
- LangSmith for traces + cost
|
| 241 |
+
- PagerDuty for alert routing
|
| 242 |
+
- A real prompt registry (MLflow or custom)
|
| 243 |
+
|
| 244 |
+
**SRE additions:**
|
| 245 |
+
- Anomaly detection on all metrics (burn rate, cost spike, eval regression)
|
| 246 |
+
- Alert severity: critical pages immediately, warning creates ticket
|
| 247 |
+
- Error budget burn rate tracked and projected
|
| 248 |
+
- Prompt version tied to eval score β shows which prompt version caused regression
|
| 249 |
+
""")
|
| 250 |
+
|
| 251 |
+
gr.Markdown("""
|
| 252 |
+
---
|
| 253 |
+
[GitHub](https://github.com/amarshiv86/p10-observability) Β·
|
| 254 |
+
[P09 Eval](https://huggingface.co/spaces/amarshiv86/p09-llm-eval) Β·
|
| 255 |
+
[P08 Agent](https://huggingface.co/spaces/amarshiv86/p08-sre-agent) Β·
|
| 256 |
+
[Portfolio](https://github.com/amarshiv86)
|
| 257 |
+
""")
|
| 258 |
+
|
| 259 |
+
refresh_btn.click(
|
| 260 |
+
fn=refresh_dashboard,
|
| 261 |
+
outputs=[header_out, alerts_out, eval_out, slo_out,
|
| 262 |
+
latency_out, cost_out, prompt_out],
|
| 263 |
+
)
|
| 264 |
+
|
| 265 |
+
# Auto-load on startup
|
| 266 |
+
demo.load(
|
| 267 |
+
fn=refresh_dashboard,
|
| 268 |
+
outputs=[header_out, alerts_out, eval_out, slo_out,
|
| 269 |
+
latency_out, cost_out, prompt_out],
|
| 270 |
+
)
|
| 271 |
+
|
| 272 |
+
demo.launch()
|
requirements.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
gradio==5.29.0
|
| 2 |
+
audioop-lts
|
src/alerts.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
P10 Β· Alert Manager
|
| 3 |
+
Anomaly detection on dashboard metrics.
|
| 4 |
+
Generates structured alerts β in production would push to Slack/PagerDuty.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from dataclasses import dataclass
|
| 8 |
+
from typing import Any
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@dataclass
|
| 12 |
+
class Alert:
|
| 13 |
+
id: str
|
| 14 |
+
severity: str # critical | warning | info
|
| 15 |
+
category: str # slo | cost | latency | eval | prompt
|
| 16 |
+
service: str
|
| 17 |
+
message: str
|
| 18 |
+
value: float
|
| 19 |
+
threshold: float
|
| 20 |
+
recommendation: str
|
| 21 |
+
|
| 22 |
+
def to_dict(self) -> dict:
|
| 23 |
+
return {
|
| 24 |
+
"id": self.id,
|
| 25 |
+
"severity": self.severity,
|
| 26 |
+
"category": self.category,
|
| 27 |
+
"service": self.service,
|
| 28 |
+
"message": self.message,
|
| 29 |
+
"value": self.value,
|
| 30 |
+
"threshold": self.threshold,
|
| 31 |
+
"recommendation": self.recommendation,
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
# ββ Anomaly detectors βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 36 |
+
def check_slo_alerts(slo_data: dict) -> list[Alert]:
|
| 37 |
+
alerts = []
|
| 38 |
+
for svc, metrics in slo_data.items():
|
| 39 |
+
burn = metrics["burn_rate"]
|
| 40 |
+
if burn > 14.4:
|
| 41 |
+
alerts.append(Alert(
|
| 42 |
+
id=f"slo-{svc}-critical",
|
| 43 |
+
severity="critical",
|
| 44 |
+
category="slo",
|
| 45 |
+
service=svc,
|
| 46 |
+
message=f"Burn rate {burn}x β exhausts budget in <2h",
|
| 47 |
+
value=burn,
|
| 48 |
+
threshold=14.4,
|
| 49 |
+
recommendation=f"Page on-call immediately. Check recent deploys for {svc}.",
|
| 50 |
+
))
|
| 51 |
+
elif burn > 6:
|
| 52 |
+
alerts.append(Alert(
|
| 53 |
+
id=f"slo-{svc}-warning",
|
| 54 |
+
severity="warning",
|
| 55 |
+
category="slo",
|
| 56 |
+
service=svc,
|
| 57 |
+
message=f"Burn rate {burn}x β exhausts budget in <5 days",
|
| 58 |
+
value=burn,
|
| 59 |
+
threshold=6.0,
|
| 60 |
+
recommendation=f"Investigate {svc} error rate. Review slow query logs.",
|
| 61 |
+
))
|
| 62 |
+
return alerts
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def check_latency_alerts(latency_data: dict) -> list[Alert]:
|
| 66 |
+
alerts = []
|
| 67 |
+
for svc, metrics in latency_data.items():
|
| 68 |
+
p99 = metrics["p99_ms"]
|
| 69 |
+
slo = metrics["slo_ms"]
|
| 70 |
+
ratio = p99 / slo
|
| 71 |
+
if ratio > 0.9:
|
| 72 |
+
severity = "critical" if ratio > 1.0 else "warning"
|
| 73 |
+
alerts.append(Alert(
|
| 74 |
+
id=f"latency-{svc}",
|
| 75 |
+
severity=severity,
|
| 76 |
+
category="latency",
|
| 77 |
+
service=svc,
|
| 78 |
+
message=f"p99 {p99}ms is {ratio*100:.0f}% of {slo}ms SLO",
|
| 79 |
+
value=p99,
|
| 80 |
+
threshold=slo,
|
| 81 |
+
recommendation=f"Check {svc} for slow queries or resource saturation.",
|
| 82 |
+
))
|
| 83 |
+
return alerts
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def check_cost_alerts(cost_data: dict) -> list[Alert]:
|
| 87 |
+
alerts = []
|
| 88 |
+
avg = cost_data.get("avg_daily", 0)
|
| 89 |
+
daily = cost_data.get("daily", [])
|
| 90 |
+
if len(daily) >= 2:
|
| 91 |
+
today = daily[-1]["total"]
|
| 92 |
+
yesterday = daily[-2]["total"]
|
| 93 |
+
if yesterday > 0 and today > yesterday * 1.5:
|
| 94 |
+
alerts.append(Alert(
|
| 95 |
+
id="cost-spike",
|
| 96 |
+
severity="warning",
|
| 97 |
+
category="cost",
|
| 98 |
+
service="llm-costs",
|
| 99 |
+
message=f"Daily cost ${today:.4f} is 50%+ above yesterday ${yesterday:.4f}",
|
| 100 |
+
value=today,
|
| 101 |
+
threshold=yesterday * 1.5,
|
| 102 |
+
recommendation="Check for unusual traffic or prompt loops causing extra tokens.",
|
| 103 |
+
))
|
| 104 |
+
return alerts
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def check_eval_alerts(eval_history: list[dict]) -> list[Alert]:
|
| 108 |
+
alerts = []
|
| 109 |
+
if len(eval_history) >= 2:
|
| 110 |
+
current = eval_history[-1]["avg_composite"]
|
| 111 |
+
previous = eval_history[-2]["avg_composite"]
|
| 112 |
+
if previous > 0:
|
| 113 |
+
drop_pct = ((previous - current) / previous) * 100
|
| 114 |
+
if drop_pct > 10:
|
| 115 |
+
alerts.append(Alert(
|
| 116 |
+
id="eval-regression",
|
| 117 |
+
severity="critical",
|
| 118 |
+
category="eval",
|
| 119 |
+
service="llm-quality",
|
| 120 |
+
message=f"Eval score dropped {drop_pct:.1f}% ({previous:.2f} β {current:.2f})",
|
| 121 |
+
value=current,
|
| 122 |
+
threshold=previous * 0.9,
|
| 123 |
+
recommendation="Block deployment. Review recent prompt changes or model updates.",
|
| 124 |
+
))
|
| 125 |
+
elif drop_pct > 5:
|
| 126 |
+
alerts.append(Alert(
|
| 127 |
+
id="eval-warning",
|
| 128 |
+
severity="warning",
|
| 129 |
+
category="eval",
|
| 130 |
+
service="llm-quality",
|
| 131 |
+
message=f"Eval score dropped {drop_pct:.1f}% β approaching regression threshold",
|
| 132 |
+
value=current,
|
| 133 |
+
threshold=previous * 0.95,
|
| 134 |
+
recommendation="Investigate prompt changes. Run extended test suite.",
|
| 135 |
+
))
|
| 136 |
+
return alerts
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def detect_all_anomalies(dashboard_data: dict) -> list[Alert]:
|
| 140 |
+
"""Run all anomaly detectors and return combined alert list."""
|
| 141 |
+
alerts = []
|
| 142 |
+
alerts += check_slo_alerts(dashboard_data.get("slo", {}))
|
| 143 |
+
alerts += check_latency_alerts(dashboard_data.get("latency", {}))
|
| 144 |
+
alerts += check_cost_alerts(dashboard_data.get("cost_metrics", {}))
|
| 145 |
+
alerts += check_eval_alerts(dashboard_data.get("eval_history", []))
|
| 146 |
+
|
| 147 |
+
# Sort: critical first, then warning, then info
|
| 148 |
+
priority = {"critical": 0, "warning": 1, "info": 2}
|
| 149 |
+
return sorted(alerts, key=lambda a: priority.get(a.severity, 3))
|
src/collector.py
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
P10 Β· Observability Dashboard β Data Collector
|
| 3 |
+
Pulls metrics from:
|
| 4 |
+
- P09 SQLite eval DB (real)
|
| 5 |
+
- Mock LLM cost tracker
|
| 6 |
+
- Mock latency metrics
|
| 7 |
+
- Mock SLO burn rates (reuses P08 pattern)
|
| 8 |
+
- Prompt version store (local JSON)
|
| 9 |
+
- Mock alert feed
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
import json
|
| 13 |
+
import os
|
| 14 |
+
import random
|
| 15 |
+
import sqlite3
|
| 16 |
+
import time
|
| 17 |
+
from datetime import datetime, timedelta, timezone
|
| 18 |
+
from typing import Any
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
# ββ Helpers βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 22 |
+
def now_utc() -> str:
|
| 23 |
+
return datetime.now(timezone.utc).isoformat()
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def past(hours: int) -> str:
|
| 27 |
+
return (datetime.now(timezone.utc) - timedelta(hours=hours)).isoformat()
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def past_days(days: int) -> str:
|
| 31 |
+
return (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
# ββ P09 Eval scores (real SQLite) βββββββββββββββββββββββββββββββββββββββββββββ
|
| 35 |
+
P09_DB_PATH = os.environ.get(
|
| 36 |
+
"P09_DB_PATH",
|
| 37 |
+
os.path.join(os.path.dirname(__file__), "..", "data", "processed", "eval_history.db"),
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
MOCK_EVAL_HISTORY = [
|
| 41 |
+
{"run_id": f"run_{i:03d}", "timestamp": past_days(30 - i * 2),
|
| 42 |
+
"avg_composite": round(5.5 + i * 0.18 + random.uniform(-0.2, 0.2), 2),
|
| 43 |
+
"pass_rate": round(60 + i * 2.5 + random.uniform(-3, 3), 1),
|
| 44 |
+
"model_name": "qwen2.5-0.5b"}
|
| 45 |
+
for i in range(15)
|
| 46 |
+
]
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def get_eval_history(n: int = 15) -> list[dict]:
|
| 50 |
+
"""Get eval score history from P09 SQLite DB. Falls back to mock."""
|
| 51 |
+
db_path = os.path.abspath(P09_DB_PATH)
|
| 52 |
+
try:
|
| 53 |
+
if os.path.exists(db_path):
|
| 54 |
+
conn = sqlite3.connect(db_path)
|
| 55 |
+
conn.row_factory = sqlite3.Row
|
| 56 |
+
rows = conn.execute("""
|
| 57 |
+
SELECT run_id, timestamp, avg_composite, pass_rate, model_name
|
| 58 |
+
FROM eval_runs ORDER BY timestamp DESC LIMIT ?
|
| 59 |
+
""", (n,)).fetchall()
|
| 60 |
+
conn.close()
|
| 61 |
+
if rows:
|
| 62 |
+
return [dict(r) for r in reversed(rows)]
|
| 63 |
+
except Exception:
|
| 64 |
+
pass
|
| 65 |
+
return MOCK_EVAL_HISTORY[-n:]
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def get_latest_eval() -> dict:
|
| 69 |
+
history = get_eval_history(1)
|
| 70 |
+
return history[-1] if history else {"avg_composite": 0, "pass_rate": 0}
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
# ββ LLM Cost tracker (mock) βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 74 |
+
MODELS = ["qwen2.5-0.5b", "phi-3-mini", "mistral-7b"]
|
| 75 |
+
MODEL_COSTS = {
|
| 76 |
+
"qwen2.5-0.5b": {"input_per_1k": 0.00, "output_per_1k": 0.00},
|
| 77 |
+
"phi-3-mini": {"input_per_1k": 0.001, "output_per_1k": 0.002},
|
| 78 |
+
"mistral-7b": {"input_per_1k": 0.002, "output_per_1k": 0.004},
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def get_cost_metrics(days: int = 7) -> dict:
|
| 83 |
+
"""Mock LLM cost breakdown by model and day."""
|
| 84 |
+
daily = []
|
| 85 |
+
for d in range(days):
|
| 86 |
+
date = past_days(days - d - 1)
|
| 87 |
+
day_cost = {}
|
| 88 |
+
for model in MODELS:
|
| 89 |
+
requests = random.randint(20, 150)
|
| 90 |
+
avg_input = random.randint(200, 800)
|
| 91 |
+
avg_output = random.randint(50, 300)
|
| 92 |
+
costs = MODEL_COSTS[model]
|
| 93 |
+
cost = (
|
| 94 |
+
requests * avg_input / 1000 * costs["input_per_1k"]
|
| 95 |
+
+ requests * avg_output / 1000 * costs["output_per_1k"]
|
| 96 |
+
)
|
| 97 |
+
day_cost[model] = round(cost, 4)
|
| 98 |
+
daily.append({"date": date, **day_cost,
|
| 99 |
+
"total": round(sum(day_cost.values()), 4)})
|
| 100 |
+
|
| 101 |
+
total_7d = round(sum(d["total"] for d in daily), 4)
|
| 102 |
+
return {
|
| 103 |
+
"daily": daily,
|
| 104 |
+
"total_7d": total_7d,
|
| 105 |
+
"avg_daily": round(total_7d / days, 4),
|
| 106 |
+
"by_model": {
|
| 107 |
+
m: round(sum(d[m] for d in daily), 4) for m in MODELS
|
| 108 |
+
},
|
| 109 |
+
}
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
# ββ Latency metrics (mock p50/p95/p99) βββββββββββββββββββββββββββββββββββββββ
|
| 113 |
+
SERVICES_LATENCY = {
|
| 114 |
+
"p06-code-review": {"p50": 1200, "p95": 3400, "p99": 5800, "slo_ms": 30000},
|
| 115 |
+
"p08-sre-agent": {"p50": 800, "p95": 2100, "p99": 4200, "slo_ms": 10000},
|
| 116 |
+
"p09-eval": {"p50": 45, "p95": 120, "p99": 280, "slo_ms": 5000},
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def get_latency_metrics() -> dict:
|
| 121 |
+
services = {}
|
| 122 |
+
for svc, base in SERVICES_LATENCY.items():
|
| 123 |
+
jitter = lambda x: int(x * random.uniform(0.85, 1.15))
|
| 124 |
+
p50 = jitter(base["p50"])
|
| 125 |
+
p95 = jitter(base["p95"])
|
| 126 |
+
p99 = jitter(base["p99"])
|
| 127 |
+
services[svc] = {
|
| 128 |
+
"p50_ms": p50,
|
| 129 |
+
"p95_ms": p95,
|
| 130 |
+
"p99_ms": p99,
|
| 131 |
+
"slo_ms": base["slo_ms"],
|
| 132 |
+
"slo_ok": p99 < base["slo_ms"],
|
| 133 |
+
"trend": random.choice(["stable", "stable", "improving", "degrading"]),
|
| 134 |
+
}
|
| 135 |
+
return services
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
# ββ SLO burn rates (reuses P08 mock pattern) ββββββββββββββββββββββββοΏ½οΏ½οΏ½βββββββββ
|
| 139 |
+
SLO_SERVICES = {
|
| 140 |
+
"api-gateway": {"slo": 99.9, "error_rate": 0.08},
|
| 141 |
+
"payment-service": {"slo": 99.95, "error_rate": 0.02},
|
| 142 |
+
"auth-service": {"slo": 99.9, "error_rate": 0.45},
|
| 143 |
+
"notification-service": {"slo": 99.5, "error_rate": 0.03},
|
| 144 |
+
}
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def get_slo_metrics() -> dict:
|
| 148 |
+
services = {}
|
| 149 |
+
for svc, cfg in SLO_SERVICES.items():
|
| 150 |
+
budget_pct = 1 - (cfg["slo"] / 100)
|
| 151 |
+
burn_rate = cfg["error_rate"] / (budget_pct * 100)
|
| 152 |
+
jitter = random.uniform(0.9, 1.1)
|
| 153 |
+
burn_rate = round(burn_rate * jitter, 2)
|
| 154 |
+
|
| 155 |
+
if burn_rate > 14.4:
|
| 156 |
+
status = "CRITICAL"
|
| 157 |
+
elif burn_rate > 6:
|
| 158 |
+
status = "WARNING"
|
| 159 |
+
elif burn_rate > 1:
|
| 160 |
+
status = "ELEVATED"
|
| 161 |
+
else:
|
| 162 |
+
status = "OK"
|
| 163 |
+
|
| 164 |
+
services[svc] = {
|
| 165 |
+
"slo_pct": cfg["slo"],
|
| 166 |
+
"error_rate_pct": round(cfg["error_rate"] * jitter, 3),
|
| 167 |
+
"burn_rate": burn_rate,
|
| 168 |
+
"status": status,
|
| 169 |
+
"budget_remaining_pct": round(max(0, 100 - burn_rate * 2), 1),
|
| 170 |
+
}
|
| 171 |
+
return services
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
# ββ Prompt version history ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 175 |
+
PROMPT_VERSIONS = [
|
| 176 |
+
{"version": "v1.0", "date": past_days(30), "project": "p06-code-review",
|
| 177 |
+
"description": "Initial prompt β basic severity classification",
|
| 178 |
+
"avg_score": 5.2, "requests": 120},
|
| 179 |
+
{"version": "v1.1", "date": past_days(22), "project": "p06-code-review",
|
| 180 |
+
"description": "Added JSON output format requirement",
|
| 181 |
+
"avg_score": 6.4, "requests": 340},
|
| 182 |
+
{"version": "v1.2", "date": past_days(14), "project": "p06-code-review",
|
| 183 |
+
"description": "Added category field and severity guide",
|
| 184 |
+
"avg_score": 7.1, "requests": 520},
|
| 185 |
+
{"version": "v1.0", "date": past_days(20), "project": "p08-sre-agent",
|
| 186 |
+
"description": "Initial ReAct prompt with 5 tools",
|
| 187 |
+
"avg_score": 4.8, "requests": 85},
|
| 188 |
+
{"version": "v1.1", "date": past_days(10), "project": "p08-sre-agent",
|
| 189 |
+
"description": "Added tool output format examples",
|
| 190 |
+
"avg_score": 6.2, "requests": 210},
|
| 191 |
+
{"version": "v1.0", "date": past_days(7), "project": "p09-eval",
|
| 192 |
+
"description": "LLM judge prompt with 0-10 scale",
|
| 193 |
+
"avg_score": 6.8, "requests": 450},
|
| 194 |
+
]
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def get_prompt_versions() -> list[dict]:
|
| 198 |
+
return sorted(PROMPT_VERSIONS, key=lambda x: x["date"], reverse=True)
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
# ββ Alert feed ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 202 |
+
MOCK_ALERTS = [
|
| 203 |
+
{"id": "ALT-001", "severity": "critical", "service": "auth-service",
|
| 204 |
+
"message": "SLO burn rate 4.5x β error budget at risk",
|
| 205 |
+
"fired_at": past(2), "status": "firing"},
|
| 206 |
+
{"id": "ALT-002", "severity": "warning", "service": "api-gateway",
|
| 207 |
+
"message": "p99 latency 3400ms approaching 30s SLO",
|
| 208 |
+
"fired_at": past(1), "status": "firing"},
|
| 209 |
+
{"id": "ALT-003", "severity": "warning", "service": "p09-eval",
|
| 210 |
+
"message": "Eval pass rate dropped from 90% to 80%",
|
| 211 |
+
"fired_at": past(4), "status": "resolved"},
|
| 212 |
+
{"id": "ALT-004", "severity": "info", "service": "payment-service",
|
| 213 |
+
"message": "Certificate expiring in 7 days",
|
| 214 |
+
"fired_at": past(18), "status": "firing"},
|
| 215 |
+
]
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def get_alerts() -> list[dict]:
|
| 219 |
+
return MOCK_ALERTS
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
# ββ Full dashboard snapshot βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 223 |
+
def get_dashboard_data() -> dict[str, Any]:
|
| 224 |
+
"""Single call that returns all dashboard panels."""
|
| 225 |
+
return {
|
| 226 |
+
"timestamp": now_utc(),
|
| 227 |
+
"eval_history": get_eval_history(),
|
| 228 |
+
"latest_eval": get_latest_eval(),
|
| 229 |
+
"cost_metrics": get_cost_metrics(),
|
| 230 |
+
"latency": get_latency_metrics(),
|
| 231 |
+
"slo": get_slo_metrics(),
|
| 232 |
+
"prompt_versions": get_prompt_versions(),
|
| 233 |
+
"alerts": get_alerts(),
|
| 234 |
+
}
|