amarshiv86 commited on
Commit
df6ae54
Β·
verified Β·
1 Parent(s): 37221ec

Upload folder using huggingface_hub

Browse files
Files changed (5) hide show
  1. README.md +13 -6
  2. app.py +272 -0
  3. requirements.txt +2 -0
  4. src/alerts.py +149 -0
  5. src/collector.py +234 -0
README.md CHANGED
@@ -1,13 +1,20 @@
1
  ---
2
- title: P10 Observability
3
- emoji: πŸ“Š
4
  colorFrom: indigo
5
- colorTo: green
6
  sdk: gradio
7
- sdk_version: 6.17.3
8
- python_version: '3.13'
9
  app_file: app.py
10
  pinned: false
11
  ---
12
 
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: P10 LLM Observability Dashboard
3
+ emoji: πŸ–₯️
4
  colorFrom: indigo
5
+ colorTo: purple
6
  sdk: gradio
7
+ sdk_version: 5.29.0
 
8
  app_file: app.py
9
  pinned: false
10
  ---
11
 
12
+ # P10 Β· LLM Observability Dashboard
13
+
14
+ Unified observability for LLM systems β€” eval scores, SLO burn rates,
15
+ latency percentiles, cost tracking, prompt versioning, and anomaly alerts.
16
+
17
+ Ties together P06 (code review), P08 (SRE agent), P09 (eval framework).
18
+ Part of the [Staff SRE Β· AI Engineer Portfolio](https://github.com/amarshiv86).
19
+
20
+ > Auto-loads on startup. Click Refresh to update.
app.py ADDED
@@ -0,0 +1,272 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ P10 Β· Observability Dashboard β€” HuggingFace Space
3
+ Full dashboard: eval scores, costs, latency, SLOs, prompt versions, alerts.
4
+ gradio==5.29.0 + audioop-lts for Python 3.13 compatibility.
5
+ """
6
+
7
+ import os
8
+ import sys
9
+
10
+ import gradio as gr
11
+
12
+ sys.path.insert(0, os.path.dirname(__file__))
13
+ from src.collector import get_dashboard_data
14
+ from src.alerts import detect_all_anomalies
15
+
16
+ SEVERITY_EMOJI = {
17
+ "critical": "πŸ”΄",
18
+ "warning": "🟠",
19
+ "info": "πŸ”΅",
20
+ "OK": "βœ…",
21
+ "ELEVATED": "🟑",
22
+ "WARNING": "🟠",
23
+ "CRITICAL": "πŸ”΄",
24
+ }
25
+
26
+ STATUS_COLOR = {
27
+ "OK": "#22d3a0",
28
+ "ELEVATED": "#f59e0b",
29
+ "WARNING": "#f97316",
30
+ "CRITICAL": "#ef4444",
31
+ }
32
+
33
+
34
+ def build_eval_panel(data: dict) -> str:
35
+ history = data["eval_history"]
36
+ latest = data["latest_eval"]
37
+ score = latest.get("avg_composite", 0)
38
+ pass_rate = latest.get("pass_rate", 0)
39
+
40
+ trend = ""
41
+ if len(history) >= 2:
42
+ prev = history[-2]["avg_composite"]
43
+ curr = history[-1]["avg_composite"]
44
+ delta = curr - prev
45
+ trend = f" ({'↑' if delta >= 0 else '↓'}{abs(delta):.2f})"
46
+
47
+ lines = [
48
+ f"## πŸ“Š Eval Score Trend",
49
+ f"**Latest:** {score:.2f}/10{trend} Β· Pass rate: {pass_rate}%",
50
+ "",
51
+ "| Run | Date | Score | Pass Rate |",
52
+ "|-----|------|-------|-----------|",
53
+ ]
54
+ for r in history[-8:]:
55
+ bar = "β–ˆ" * int(r["avg_composite"]) + "β–‘" * (10 - int(r["avg_composite"]))
56
+ lines.append(
57
+ f"| `{r['run_id'][-6:]}` "
58
+ f"| {r['timestamp'][:10]} "
59
+ f"| `{bar}` {r['avg_composite']:.2f} "
60
+ f"| {r.get('pass_rate', 0):.0f}% |"
61
+ )
62
+ return "\n".join(lines)
63
+
64
+
65
+ def build_slo_panel(data: dict) -> str:
66
+ slo = data["slo"]
67
+ lines = ["## 🎯 SLO Status"]
68
+ lines.append("")
69
+ lines.append("| Service | SLO | Error Rate | Burn Rate | Budget Left | Status |")
70
+ lines.append("|---------|-----|------------|-----------|-------------|--------|")
71
+ for svc, m in slo.items():
72
+ emoji = SEVERITY_EMOJI.get(m["status"], "❓")
73
+ lines.append(
74
+ f"| `{svc}` "
75
+ f"| {m['slo_pct']}% "
76
+ f"| {m['error_rate_pct']}% "
77
+ f"| {m['burn_rate']}x "
78
+ f"| {m['budget_remaining_pct']}% "
79
+ f"| {emoji} {m['status']} |"
80
+ )
81
+ return "\n".join(lines)
82
+
83
+
84
+ def build_latency_panel(data: dict) -> str:
85
+ lat = data["latency"]
86
+ lines = ["## ⚑ Latency (p50 / p95 / p99)"]
87
+ lines.append("")
88
+ lines.append("| Service | p50 | p95 | p99 | SLO | Status |")
89
+ lines.append("|---------|-----|-----|-----|-----|--------|")
90
+ for svc, m in lat.items():
91
+ ok = "βœ… OK" if m["slo_ok"] else "❌ BREACH"
92
+ lines.append(
93
+ f"| `{svc}` "
94
+ f"| {m['p50_ms']}ms "
95
+ f"| {m['p95_ms']}ms "
96
+ f"| {m['p99_ms']}ms "
97
+ f"| {m['slo_ms']}ms "
98
+ f"| {ok} |"
99
+ )
100
+ return "\n".join(lines)
101
+
102
+
103
+ def build_cost_panel(data: dict) -> str:
104
+ cost = data["cost_metrics"]
105
+ daily = cost["daily"]
106
+ lines = [
107
+ "## πŸ’° LLM Cost Tracker (7 days)",
108
+ "",
109
+ f"**Total 7d:** ${cost['total_7d']:.4f} Β· "
110
+ f"**Avg daily:** ${cost['avg_daily']:.4f}",
111
+ "",
112
+ "| Date | qwen2.5-0.5b | phi-3-mini | mistral-7b | Total |",
113
+ "|------|-------------|------------|------------|-------|",
114
+ ]
115
+ for d in daily:
116
+ lines.append(
117
+ f"| {d['date']} "
118
+ f"| ${d.get('qwen2.5-0.5b', 0):.4f} "
119
+ f"| ${d.get('phi-3-mini', 0):.4f} "
120
+ f"| ${d.get('mistral-7b', 0):.4f} "
121
+ f"| **${d['total']:.4f}** |"
122
+ )
123
+ lines.append("")
124
+ lines.append("**By model:**")
125
+ for model, cost_val in cost["by_model"].items():
126
+ lines.append(f"- `{model}`: ${cost_val:.4f}")
127
+ return "\n".join(lines)
128
+
129
+
130
+ def build_prompt_panel(data: dict) -> str:
131
+ versions = data["prompt_versions"]
132
+ lines = [
133
+ "## πŸ“ Prompt Version History",
134
+ "",
135
+ "| Version | Date | Project | Description | Avg Score | Requests |",
136
+ "|---------|------|---------|-------------|-----------|----------|",
137
+ ]
138
+ for v in versions:
139
+ score_bar = "β–ˆ" * int(v["avg_score"]) + "β–‘" * (10 - int(v["avg_score"]))
140
+ lines.append(
141
+ f"| `{v['version']}` "
142
+ f"| {v['date'][:10]} "
143
+ f"| `{v['project']}` "
144
+ f"| {v['description']} "
145
+ f"| `{score_bar}` {v['avg_score']} "
146
+ f"| {v['requests']} |"
147
+ )
148
+ return "\n".join(lines)
149
+
150
+
151
+ def build_alerts_panel(alerts: list) -> str:
152
+ if not alerts:
153
+ return "## πŸ”” Alerts\n\nβœ… **No anomalies detected**"
154
+
155
+ lines = [f"## πŸ”” Alerts ({len(alerts)} active)", ""]
156
+ for a in alerts:
157
+ emoji = SEVERITY_EMOJI.get(a.severity, "❓")
158
+ lines += [
159
+ f"### {emoji} `{a.severity.upper()}` β€” {a.service}",
160
+ f"**{a.message}**",
161
+ f"- Value: `{a.value}` Β· Threshold: `{a.threshold}`",
162
+ f"- πŸ’‘ {a.recommendation}",
163
+ "",
164
+ ]
165
+ return "\n".join(lines)
166
+
167
+
168
+ def refresh_dashboard() -> tuple:
169
+ """Fetch all data and build all panels."""
170
+ data = get_dashboard_data()
171
+ anomalies = detect_all_anomalies(data)
172
+
173
+ alert_count = len([a for a in anomalies if a.severity == "critical"])
174
+ warning_count = len([a for a in anomalies if a.severity == "warning"])
175
+
176
+ header = (
177
+ f"## πŸ–₯️ LLM Observability Dashboard\n"
178
+ f"**Last updated:** {data['timestamp'][:19]} UTC Β· "
179
+ f"πŸ”΄ {alert_count} critical Β· 🟠 {warning_count} warnings\n\n"
180
+ f"_P10 Β· Staff SRE + AI Engineer Portfolio β€” ties together P06, P08, P09_"
181
+ )
182
+
183
+ return (
184
+ header,
185
+ build_alerts_panel(anomalies),
186
+ build_eval_panel(data),
187
+ build_slo_panel(data),
188
+ build_latency_panel(data),
189
+ build_cost_panel(data),
190
+ build_prompt_panel(data),
191
+ )
192
+
193
+
194
+ # ── Gradio UI ──────────────────────────────────────────────────────────────────
195
+ with gr.Blocks(title="P10 Β· Observability Dashboard", theme=gr.themes.Soft()) as demo:
196
+
197
+ gr.Markdown("""
198
+ # πŸ–₯️ P10 Β· LLM Observability Dashboard
199
+ **Staff SRE + AI Engineer Portfolio**
200
+
201
+ Unified observability for LLM systems β€” eval scores (P09), SLO burn rates (P08 pattern),
202
+ latency percentiles, cost tracking, prompt versioning, and anomaly alerts.
203
+
204
+ Click **Refresh** to load live data.
205
+ """)
206
+
207
+ refresh_btn = gr.Button("πŸ”„ Refresh Dashboard", variant="primary", size="lg")
208
+
209
+ header_out = gr.Markdown()
210
+
211
+ with gr.Row():
212
+ with gr.Column():
213
+ alerts_out = gr.Markdown()
214
+ with gr.Column():
215
+ eval_out = gr.Markdown()
216
+
217
+ with gr.Row():
218
+ with gr.Column():
219
+ slo_out = gr.Markdown()
220
+ with gr.Column():
221
+ latency_out = gr.Markdown()
222
+
223
+ cost_out = gr.Markdown()
224
+ prompt_out = gr.Markdown()
225
+
226
+ with gr.Accordion("πŸ“– Architecture", open=False):
227
+ gr.Markdown("""
228
+ ## How P10 connects to the rest of the portfolio
229
+
230
+ | Data source | Project | What it provides |
231
+ |-------------|---------|-----------------|
232
+ | SQLite eval DB | P09 | Score history + regression tracking |
233
+ | Mock Prometheus | P08 pattern | SLO burn rates + error budget |
234
+ | Mock cost tracker | P10 | Token cost by model per day |
235
+ | Mock Jaeger | P10 | Latency p50/p95/p99 per service |
236
+ | Prompt store | P10 | Version history + score per version |
237
+
238
+ **In production** you would replace mocks with:
239
+ - Prometheus for SLO/latency
240
+ - LangSmith for traces + cost
241
+ - PagerDuty for alert routing
242
+ - A real prompt registry (MLflow or custom)
243
+
244
+ **SRE additions:**
245
+ - Anomaly detection on all metrics (burn rate, cost spike, eval regression)
246
+ - Alert severity: critical pages immediately, warning creates ticket
247
+ - Error budget burn rate tracked and projected
248
+ - Prompt version tied to eval score β€” shows which prompt version caused regression
249
+ """)
250
+
251
+ gr.Markdown("""
252
+ ---
253
+ [GitHub](https://github.com/amarshiv86/p10-observability) Β·
254
+ [P09 Eval](https://huggingface.co/spaces/amarshiv86/p09-llm-eval) Β·
255
+ [P08 Agent](https://huggingface.co/spaces/amarshiv86/p08-sre-agent) Β·
256
+ [Portfolio](https://github.com/amarshiv86)
257
+ """)
258
+
259
+ refresh_btn.click(
260
+ fn=refresh_dashboard,
261
+ outputs=[header_out, alerts_out, eval_out, slo_out,
262
+ latency_out, cost_out, prompt_out],
263
+ )
264
+
265
+ # Auto-load on startup
266
+ demo.load(
267
+ fn=refresh_dashboard,
268
+ outputs=[header_out, alerts_out, eval_out, slo_out,
269
+ latency_out, cost_out, prompt_out],
270
+ )
271
+
272
+ demo.launch()
requirements.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ gradio==5.29.0
2
+ audioop-lts
src/alerts.py ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ P10 Β· Alert Manager
3
+ Anomaly detection on dashboard metrics.
4
+ Generates structured alerts β€” in production would push to Slack/PagerDuty.
5
+ """
6
+
7
+ from dataclasses import dataclass
8
+ from typing import Any
9
+
10
+
11
+ @dataclass
12
+ class Alert:
13
+ id: str
14
+ severity: str # critical | warning | info
15
+ category: str # slo | cost | latency | eval | prompt
16
+ service: str
17
+ message: str
18
+ value: float
19
+ threshold: float
20
+ recommendation: str
21
+
22
+ def to_dict(self) -> dict:
23
+ return {
24
+ "id": self.id,
25
+ "severity": self.severity,
26
+ "category": self.category,
27
+ "service": self.service,
28
+ "message": self.message,
29
+ "value": self.value,
30
+ "threshold": self.threshold,
31
+ "recommendation": self.recommendation,
32
+ }
33
+
34
+
35
+ # ── Anomaly detectors ─────────────────────────────────────────────────────────
36
+ def check_slo_alerts(slo_data: dict) -> list[Alert]:
37
+ alerts = []
38
+ for svc, metrics in slo_data.items():
39
+ burn = metrics["burn_rate"]
40
+ if burn > 14.4:
41
+ alerts.append(Alert(
42
+ id=f"slo-{svc}-critical",
43
+ severity="critical",
44
+ category="slo",
45
+ service=svc,
46
+ message=f"Burn rate {burn}x β€” exhausts budget in <2h",
47
+ value=burn,
48
+ threshold=14.4,
49
+ recommendation=f"Page on-call immediately. Check recent deploys for {svc}.",
50
+ ))
51
+ elif burn > 6:
52
+ alerts.append(Alert(
53
+ id=f"slo-{svc}-warning",
54
+ severity="warning",
55
+ category="slo",
56
+ service=svc,
57
+ message=f"Burn rate {burn}x β€” exhausts budget in <5 days",
58
+ value=burn,
59
+ threshold=6.0,
60
+ recommendation=f"Investigate {svc} error rate. Review slow query logs.",
61
+ ))
62
+ return alerts
63
+
64
+
65
+ def check_latency_alerts(latency_data: dict) -> list[Alert]:
66
+ alerts = []
67
+ for svc, metrics in latency_data.items():
68
+ p99 = metrics["p99_ms"]
69
+ slo = metrics["slo_ms"]
70
+ ratio = p99 / slo
71
+ if ratio > 0.9:
72
+ severity = "critical" if ratio > 1.0 else "warning"
73
+ alerts.append(Alert(
74
+ id=f"latency-{svc}",
75
+ severity=severity,
76
+ category="latency",
77
+ service=svc,
78
+ message=f"p99 {p99}ms is {ratio*100:.0f}% of {slo}ms SLO",
79
+ value=p99,
80
+ threshold=slo,
81
+ recommendation=f"Check {svc} for slow queries or resource saturation.",
82
+ ))
83
+ return alerts
84
+
85
+
86
+ def check_cost_alerts(cost_data: dict) -> list[Alert]:
87
+ alerts = []
88
+ avg = cost_data.get("avg_daily", 0)
89
+ daily = cost_data.get("daily", [])
90
+ if len(daily) >= 2:
91
+ today = daily[-1]["total"]
92
+ yesterday = daily[-2]["total"]
93
+ if yesterday > 0 and today > yesterday * 1.5:
94
+ alerts.append(Alert(
95
+ id="cost-spike",
96
+ severity="warning",
97
+ category="cost",
98
+ service="llm-costs",
99
+ message=f"Daily cost ${today:.4f} is 50%+ above yesterday ${yesterday:.4f}",
100
+ value=today,
101
+ threshold=yesterday * 1.5,
102
+ recommendation="Check for unusual traffic or prompt loops causing extra tokens.",
103
+ ))
104
+ return alerts
105
+
106
+
107
+ def check_eval_alerts(eval_history: list[dict]) -> list[Alert]:
108
+ alerts = []
109
+ if len(eval_history) >= 2:
110
+ current = eval_history[-1]["avg_composite"]
111
+ previous = eval_history[-2]["avg_composite"]
112
+ if previous > 0:
113
+ drop_pct = ((previous - current) / previous) * 100
114
+ if drop_pct > 10:
115
+ alerts.append(Alert(
116
+ id="eval-regression",
117
+ severity="critical",
118
+ category="eval",
119
+ service="llm-quality",
120
+ message=f"Eval score dropped {drop_pct:.1f}% ({previous:.2f} β†’ {current:.2f})",
121
+ value=current,
122
+ threshold=previous * 0.9,
123
+ recommendation="Block deployment. Review recent prompt changes or model updates.",
124
+ ))
125
+ elif drop_pct > 5:
126
+ alerts.append(Alert(
127
+ id="eval-warning",
128
+ severity="warning",
129
+ category="eval",
130
+ service="llm-quality",
131
+ message=f"Eval score dropped {drop_pct:.1f}% β€” approaching regression threshold",
132
+ value=current,
133
+ threshold=previous * 0.95,
134
+ recommendation="Investigate prompt changes. Run extended test suite.",
135
+ ))
136
+ return alerts
137
+
138
+
139
+ def detect_all_anomalies(dashboard_data: dict) -> list[Alert]:
140
+ """Run all anomaly detectors and return combined alert list."""
141
+ alerts = []
142
+ alerts += check_slo_alerts(dashboard_data.get("slo", {}))
143
+ alerts += check_latency_alerts(dashboard_data.get("latency", {}))
144
+ alerts += check_cost_alerts(dashboard_data.get("cost_metrics", {}))
145
+ alerts += check_eval_alerts(dashboard_data.get("eval_history", []))
146
+
147
+ # Sort: critical first, then warning, then info
148
+ priority = {"critical": 0, "warning": 1, "info": 2}
149
+ return sorted(alerts, key=lambda a: priority.get(a.severity, 3))
src/collector.py ADDED
@@ -0,0 +1,234 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ P10 Β· Observability Dashboard β€” Data Collector
3
+ Pulls metrics from:
4
+ - P09 SQLite eval DB (real)
5
+ - Mock LLM cost tracker
6
+ - Mock latency metrics
7
+ - Mock SLO burn rates (reuses P08 pattern)
8
+ - Prompt version store (local JSON)
9
+ - Mock alert feed
10
+ """
11
+
12
+ import json
13
+ import os
14
+ import random
15
+ import sqlite3
16
+ import time
17
+ from datetime import datetime, timedelta, timezone
18
+ from typing import Any
19
+
20
+
21
+ # ── Helpers ───────────────────────────────────────────────────────────────────
22
+ def now_utc() -> str:
23
+ return datetime.now(timezone.utc).isoformat()
24
+
25
+
26
+ def past(hours: int) -> str:
27
+ return (datetime.now(timezone.utc) - timedelta(hours=hours)).isoformat()
28
+
29
+
30
+ def past_days(days: int) -> str:
31
+ return (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
32
+
33
+
34
+ # ── P09 Eval scores (real SQLite) ─────────────────────────────────────────────
35
+ P09_DB_PATH = os.environ.get(
36
+ "P09_DB_PATH",
37
+ os.path.join(os.path.dirname(__file__), "..", "data", "processed", "eval_history.db"),
38
+ )
39
+
40
+ MOCK_EVAL_HISTORY = [
41
+ {"run_id": f"run_{i:03d}", "timestamp": past_days(30 - i * 2),
42
+ "avg_composite": round(5.5 + i * 0.18 + random.uniform(-0.2, 0.2), 2),
43
+ "pass_rate": round(60 + i * 2.5 + random.uniform(-3, 3), 1),
44
+ "model_name": "qwen2.5-0.5b"}
45
+ for i in range(15)
46
+ ]
47
+
48
+
49
+ def get_eval_history(n: int = 15) -> list[dict]:
50
+ """Get eval score history from P09 SQLite DB. Falls back to mock."""
51
+ db_path = os.path.abspath(P09_DB_PATH)
52
+ try:
53
+ if os.path.exists(db_path):
54
+ conn = sqlite3.connect(db_path)
55
+ conn.row_factory = sqlite3.Row
56
+ rows = conn.execute("""
57
+ SELECT run_id, timestamp, avg_composite, pass_rate, model_name
58
+ FROM eval_runs ORDER BY timestamp DESC LIMIT ?
59
+ """, (n,)).fetchall()
60
+ conn.close()
61
+ if rows:
62
+ return [dict(r) for r in reversed(rows)]
63
+ except Exception:
64
+ pass
65
+ return MOCK_EVAL_HISTORY[-n:]
66
+
67
+
68
+ def get_latest_eval() -> dict:
69
+ history = get_eval_history(1)
70
+ return history[-1] if history else {"avg_composite": 0, "pass_rate": 0}
71
+
72
+
73
+ # ── LLM Cost tracker (mock) ───────────────────────────────────────────────────
74
+ MODELS = ["qwen2.5-0.5b", "phi-3-mini", "mistral-7b"]
75
+ MODEL_COSTS = {
76
+ "qwen2.5-0.5b": {"input_per_1k": 0.00, "output_per_1k": 0.00},
77
+ "phi-3-mini": {"input_per_1k": 0.001, "output_per_1k": 0.002},
78
+ "mistral-7b": {"input_per_1k": 0.002, "output_per_1k": 0.004},
79
+ }
80
+
81
+
82
+ def get_cost_metrics(days: int = 7) -> dict:
83
+ """Mock LLM cost breakdown by model and day."""
84
+ daily = []
85
+ for d in range(days):
86
+ date = past_days(days - d - 1)
87
+ day_cost = {}
88
+ for model in MODELS:
89
+ requests = random.randint(20, 150)
90
+ avg_input = random.randint(200, 800)
91
+ avg_output = random.randint(50, 300)
92
+ costs = MODEL_COSTS[model]
93
+ cost = (
94
+ requests * avg_input / 1000 * costs["input_per_1k"]
95
+ + requests * avg_output / 1000 * costs["output_per_1k"]
96
+ )
97
+ day_cost[model] = round(cost, 4)
98
+ daily.append({"date": date, **day_cost,
99
+ "total": round(sum(day_cost.values()), 4)})
100
+
101
+ total_7d = round(sum(d["total"] for d in daily), 4)
102
+ return {
103
+ "daily": daily,
104
+ "total_7d": total_7d,
105
+ "avg_daily": round(total_7d / days, 4),
106
+ "by_model": {
107
+ m: round(sum(d[m] for d in daily), 4) for m in MODELS
108
+ },
109
+ }
110
+
111
+
112
+ # ── Latency metrics (mock p50/p95/p99) ───────────────────────────────────────
113
+ SERVICES_LATENCY = {
114
+ "p06-code-review": {"p50": 1200, "p95": 3400, "p99": 5800, "slo_ms": 30000},
115
+ "p08-sre-agent": {"p50": 800, "p95": 2100, "p99": 4200, "slo_ms": 10000},
116
+ "p09-eval": {"p50": 45, "p95": 120, "p99": 280, "slo_ms": 5000},
117
+ }
118
+
119
+
120
+ def get_latency_metrics() -> dict:
121
+ services = {}
122
+ for svc, base in SERVICES_LATENCY.items():
123
+ jitter = lambda x: int(x * random.uniform(0.85, 1.15))
124
+ p50 = jitter(base["p50"])
125
+ p95 = jitter(base["p95"])
126
+ p99 = jitter(base["p99"])
127
+ services[svc] = {
128
+ "p50_ms": p50,
129
+ "p95_ms": p95,
130
+ "p99_ms": p99,
131
+ "slo_ms": base["slo_ms"],
132
+ "slo_ok": p99 < base["slo_ms"],
133
+ "trend": random.choice(["stable", "stable", "improving", "degrading"]),
134
+ }
135
+ return services
136
+
137
+
138
+ # ── SLO burn rates (reuses P08 mock pattern) ────────────────────────���─────────
139
+ SLO_SERVICES = {
140
+ "api-gateway": {"slo": 99.9, "error_rate": 0.08},
141
+ "payment-service": {"slo": 99.95, "error_rate": 0.02},
142
+ "auth-service": {"slo": 99.9, "error_rate": 0.45},
143
+ "notification-service": {"slo": 99.5, "error_rate": 0.03},
144
+ }
145
+
146
+
147
+ def get_slo_metrics() -> dict:
148
+ services = {}
149
+ for svc, cfg in SLO_SERVICES.items():
150
+ budget_pct = 1 - (cfg["slo"] / 100)
151
+ burn_rate = cfg["error_rate"] / (budget_pct * 100)
152
+ jitter = random.uniform(0.9, 1.1)
153
+ burn_rate = round(burn_rate * jitter, 2)
154
+
155
+ if burn_rate > 14.4:
156
+ status = "CRITICAL"
157
+ elif burn_rate > 6:
158
+ status = "WARNING"
159
+ elif burn_rate > 1:
160
+ status = "ELEVATED"
161
+ else:
162
+ status = "OK"
163
+
164
+ services[svc] = {
165
+ "slo_pct": cfg["slo"],
166
+ "error_rate_pct": round(cfg["error_rate"] * jitter, 3),
167
+ "burn_rate": burn_rate,
168
+ "status": status,
169
+ "budget_remaining_pct": round(max(0, 100 - burn_rate * 2), 1),
170
+ }
171
+ return services
172
+
173
+
174
+ # ── Prompt version history ────────────────────────────────────────────────────
175
+ PROMPT_VERSIONS = [
176
+ {"version": "v1.0", "date": past_days(30), "project": "p06-code-review",
177
+ "description": "Initial prompt β€” basic severity classification",
178
+ "avg_score": 5.2, "requests": 120},
179
+ {"version": "v1.1", "date": past_days(22), "project": "p06-code-review",
180
+ "description": "Added JSON output format requirement",
181
+ "avg_score": 6.4, "requests": 340},
182
+ {"version": "v1.2", "date": past_days(14), "project": "p06-code-review",
183
+ "description": "Added category field and severity guide",
184
+ "avg_score": 7.1, "requests": 520},
185
+ {"version": "v1.0", "date": past_days(20), "project": "p08-sre-agent",
186
+ "description": "Initial ReAct prompt with 5 tools",
187
+ "avg_score": 4.8, "requests": 85},
188
+ {"version": "v1.1", "date": past_days(10), "project": "p08-sre-agent",
189
+ "description": "Added tool output format examples",
190
+ "avg_score": 6.2, "requests": 210},
191
+ {"version": "v1.0", "date": past_days(7), "project": "p09-eval",
192
+ "description": "LLM judge prompt with 0-10 scale",
193
+ "avg_score": 6.8, "requests": 450},
194
+ ]
195
+
196
+
197
+ def get_prompt_versions() -> list[dict]:
198
+ return sorted(PROMPT_VERSIONS, key=lambda x: x["date"], reverse=True)
199
+
200
+
201
+ # ── Alert feed ────────────────────────────────────────────────────────────────
202
+ MOCK_ALERTS = [
203
+ {"id": "ALT-001", "severity": "critical", "service": "auth-service",
204
+ "message": "SLO burn rate 4.5x β€” error budget at risk",
205
+ "fired_at": past(2), "status": "firing"},
206
+ {"id": "ALT-002", "severity": "warning", "service": "api-gateway",
207
+ "message": "p99 latency 3400ms approaching 30s SLO",
208
+ "fired_at": past(1), "status": "firing"},
209
+ {"id": "ALT-003", "severity": "warning", "service": "p09-eval",
210
+ "message": "Eval pass rate dropped from 90% to 80%",
211
+ "fired_at": past(4), "status": "resolved"},
212
+ {"id": "ALT-004", "severity": "info", "service": "payment-service",
213
+ "message": "Certificate expiring in 7 days",
214
+ "fired_at": past(18), "status": "firing"},
215
+ ]
216
+
217
+
218
+ def get_alerts() -> list[dict]:
219
+ return MOCK_ALERTS
220
+
221
+
222
+ # ── Full dashboard snapshot ───────────────────────────────────────────────────
223
+ def get_dashboard_data() -> dict[str, Any]:
224
+ """Single call that returns all dashboard panels."""
225
+ return {
226
+ "timestamp": now_utc(),
227
+ "eval_history": get_eval_history(),
228
+ "latest_eval": get_latest_eval(),
229
+ "cost_metrics": get_cost_metrics(),
230
+ "latency": get_latency_metrics(),
231
+ "slo": get_slo_metrics(),
232
+ "prompt_versions": get_prompt_versions(),
233
+ "alerts": get_alerts(),
234
+ }