feat(observability): add refined Prometheus rules and Grafana dashboard polish

grafana/provisioning/dashboards/nl2sql.json

@@ -1,68 +1,143 @@
 {
-  "title": "NL2SQL Copilot - Observability",
+  "title": "NL2SQL Copilot — Unified Observability",
+  "uid": "nl2sql-unified",
+  "timezone": "browser",
   "editable": true,
+  "schemaVersion": 38,
+  "version": 1,
+  "refresh": "30s",
   "panels": [
     {
+      "id": 1,
       "type": "timeseries",
       "title": "Stage p95 Latency (ms)",
+      "datasource": "Prometheus",
       "targets": [
-        {
-          "expr": "nl2sql:stage_p95_ms",
-          "legendFormat": "{{stage}}",
-          "refId": "A"
-        }
+        { "expr": "nl2sql:stage_p95_ms", "legendFormat": "{{stage}}", "refId": "A" }
       ],
       "fieldConfig": {
         "defaults": {
-          "unit": "milliseconds",
-          "decimals": 0
-        }
+          "unit": "ms",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green" },
+              { "value": 1000, "color": "orange" },
+              { "value": 2000, "color": "red" }
+            ]
+          }
+        },
+        "overrides": []
       },
-      "id": 1
+      "gridPos": { "x": 0, "y": 0, "w": 12, "h": 6 }
     },
     {
-      "type": "timeseries",
-      "title": "Pipeline Success Ratio",
+      "id": 2,
+      "type": "stat",
+      "title": "Pipeline OK Ratio (%)",
+      "datasource": "Prometheus",
+      "targets": [
+        { "expr": "nl2sql:pipeline_success_ratio * 100", "refId": "A" }
+      ],
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"] },
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "center"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "decimals": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red" },
+              { "value": 90, "color": "orange" },
+              { "value": 97, "color": "green" }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": { "x": 12, "y": 0, "w": 6, "h": 4 }
+    },
+    {
+      "id": 3,
+      "type": "stat",
+      "title": "Repair Success Rate (%)",
+      "datasource": "Prometheus",
       "targets": [
-        {
-          "expr": "nl2sql:pipeline_success_ratio",
-          "legendFormat": "success ratio",
-          "refId": "B"
+        { "expr": "nl2sql:repair_success_rate * 100", "refId": "A" }
+      ],
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"] },
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "center"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "decimals": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red" },
+              { "value": 80, "color": "orange" },
+              { "value": 95, "color": "green" }
+            ]
+          }
         }
+      },
+      "gridPos": { "x": 12, "y": 4, "w": 6, "h": 4 }
+    },
+    {
+      "id": 4,
+      "type": "gauge",
+      "title": "Cache Hit Ratio",
+      "datasource": "Prometheus",
+      "targets": [
+        { "expr": "nl2sql:cache_hit_ratio * 100", "refId": "A" }
       ],
       "fieldConfig": {
         "defaults": {
+          "unit": "percent",
           "min": 0,
-          "max": 1,
-          "decimals": 2
-        }
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red" },
+              { "value": 70, "color": "orange" },
+              { "value": 90, "color": "green" }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "orientation": "auto",
+        "reduceOptions": { "calcs": ["lastNotNull"] },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
       },
-      "id": 2
+      "gridPos": { "x": 0, "y": 6, "w": 6, "h": 4 }
     },
     {
+      "id": 5,
       "type": "timeseries",
-      "title": "Safety & Verifier Events",
+      "title": "Safety & Verifier Events (per min)",
+      "datasource": "Prometheus",
       "targets": [
-        {
-          "expr": "rate(safety_blocks_total[5m])",
-          "legendFormat": "safety blocks/min",
-          "refId": "C"
-        },
-        {
-          "expr": "rate(verifier_failures_total[5m])",
-          "legendFormat": "verifier failures/min",
-          "refId": "D"
-        }
+        { "expr": "rate(safety_blocks_total[5m]) * 60", "legendFormat": "safety blocks", "refId": "A" },
+        { "expr": "rate(verifier_failures_total[5m]) * 60", "legendFormat": "verifier failures", "refId": "B" }
       ],
       "fieldConfig": {
-        "defaults": {
-          "min": 0
-        }
+        "defaults": { "min": 0, "decimals": 0 }
       },
-      "id": 3
+      "gridPos": { "x": 6, "y": 6, "w": 12, "h": 6 }
     }
-  ],
-  "schemaVersion": 38,
-  "version": 1,
-  "refresh": "30s"
+  ]
 }

prometheus/grafana_dashboard.json

@@ -1,10 +1,15 @@
 {
   "title": "NL2SQL Copilot - Observability",
   "editable": true,
+  "schemaVersion": 38,
+  "version": 2,
+  "refresh": "30s",
   "panels": [
     {
+      "id": 1,
       "type": "timeseries",
       "title": "Stage p95 Latency (ms)",
+      "datasource": "Prometheus",
       "targets": [
         {
           "expr": "nl2sql:stage_p95_ms",
@@ -14,55 +19,148 @@
       ],
       "fieldConfig": {
         "defaults": {
-          "unit": "milliseconds",
-          "decimals": 0
+          "unit": "ms",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green" },
+              { "value": 1000, "color": "orange" },
+              { "value": 2000, "color": "red" }
+            ]
+          }
         }
       },
-      "id": 1
+      "gridPos": { "x": 0, "y": 0, "w": 12, "h": 6 }
     },
     {
-      "type": "timeseries",
-      "title": "Pipeline Success Ratio",
+      "id": 2,
+      "type": "stat",
+      "title": "Pipeline OK Ratio (%)",
+      "datasource": "Prometheus",
       "targets": [
         {
-          "expr": "nl2sql:pipeline_success_ratio",
-          "legendFormat": "success ratio",
+          "expr": "nl2sql:pipeline_success_ratio * 100",
+          "legendFormat": "OK ratio",
           "refId": "B"
         }
       ],
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"] },
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "center"
+      },
       "fieldConfig": {
         "defaults": {
+          "unit": "percent",
+          "decimals": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red" },
+              { "value": 90, "color": "orange" },
+              { "value": 97, "color": "green" }
+            ]
+          }
+        }
+      },
+      "gridPos": { "x": 12, "y": 0, "w": 6, "h": 4 }
+    },
+    {
+      "id": 3,
+      "type": "stat",
+      "title": "Repair Success Rate (%)",
+      "datasource": "Prometheus",
+      "targets": [
+        {
+          "expr": "nl2sql:repair_success_rate * 100",
+          "legendFormat": "repair success",
+          "refId": "C"
+        }
+      ],
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"] },
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "center"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "decimals": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red" },
+              { "value": 80, "color": "orange" },
+              { "value": 95, "color": "green" }
+            ]
+          }
+        }
+      },
+      "gridPos": { "x": 12, "y": 4, "w": 6, "h": 4 }
+    },
+    {
+      "id": 4,
+      "type": "gauge",
+      "title": "Cache Hit Ratio",
+      "datasource": "Prometheus",
+      "targets": [
+        {
+          "expr": "nl2sql:cache_hit_ratio * 100",
+          "legendFormat": "cache hit",
+          "refId": "D"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
           "min": 0,
-          "max": 1,
-          "decimals": 2
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red" },
+              { "value": 70, "color": "orange" },
+              { "value": 90, "color": "green" }
+            ]
+          }
         }
       },
-      "id": 2
+      "options": {
+        "orientation": "auto",
+        "reduceOptions": { "calcs": ["lastNotNull"] },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "gridPos": { "x": 0, "y": 6, "w": 6, "h": 4 }
     },
     {
+      "id": 5,
       "type": "timeseries",
       "title": "Safety & Verifier Events",
+      "datasource": "Prometheus",
       "targets": [
         {
-          "expr": "rate(safety_blocks_total[5m])",
+          "expr": "rate(safety_blocks_total[5m]) * 60",
           "legendFormat": "safety blocks/min",
-          "refId": "C"
+          "refId": "E"
         },
         {
-          "expr": "rate(verifier_failures_total[5m])",
+          "expr": "rate(verifier_failures_total[5m]) * 60",
           "legendFormat": "verifier failures/min",
-          "refId": "D"
+          "refId": "F"
         }
       ],
       "fieldConfig": {
         "defaults": {
-          "min": 0
+          "unit": "none",
+          "min": 0,
+          "decimals": 0
         }
       },
-      "id": 3
+      "gridPos": { "x": 6, "y": 6, "w": 12, "h": 6 }
     }
-  ],
-  "schemaVersion": 38,
-  "version": 1,
-  "refresh": "30s"
+  ]
 }

prometheus/rules.yml

@@ -1,49 +1,75 @@
 groups:
-- name: nl2sql_latency
+# 1) Recording rules (all derived metric calculations)
+- name: nl2sql_derived
+  interval: 15s
   rules:
-  # p95 latency per stage (5-minute window)
-  - record: nl2sql:stage_p95_ms
-    expr: |
-      histogram_quantile(
-        0.95,
-        sum(rate(stage_duration_ms_bucket[5m])) by (le, stage)
-      )
+    # p95 latency per stage (ms) — remove *1000 if histogram buckets are already in milliseconds
+    - record: nl2sql:stage_p95_ms
+      expr: |
+        histogram_quantile(
+          0.95,
+          sum by (le, stage) (rate(stage_duration_ms_bucket[5m]))
+        ) * 1000
 
-  # pipeline success ratio (5-minute rolling window)
-  - record: nl2sql:pipeline_success_ratio
-    expr: |
-      sum(rate(pipeline_runs_total{status="ok"}[5m]))
-      /
-      sum(rate(pipeline_runs_total[5m]))
+    # pipeline success ratio (0..1) — safe division to avoid divide-by-zero
+    - record: nl2sql:pipeline_success_ratio
+      expr: |
+        (
+          sum(rate(pipeline_runs_total{status="ok"}[5m]))
+        )
+        /
+        clamp_min(sum(rate(pipeline_runs_total[5m])), 1)
 
+    # repair success rate (0..1)
+    - record: nl2sql:repair_success_rate
+      expr: |
+        (
+          sum(rate(repair_attempts_total{outcome="success"}[5m]))
+        )
+        /
+        clamp_min(sum(rate(repair_attempts_total[5m])), 1)
+
+    # cache hit ratio (0..1)
+    - record: nl2sql:cache_hit_ratio
+      expr: |
+        (
+          sum(rate(cache_hits_total[5m]))
+        )
+        /
+        clamp_min(
+          sum(rate(cache_hits_total[5m])) + sum(rate(cache_misses_total[5m])),
+          1
+        )
+
+# 2) Alerts (must come after recording rules)
 - name: nl2sql_alerts
   rules:
-  # Alert: success ratio below 90% for 10 minutes
-  - alert: PipelineLowSuccessRatio
-    expr: nl2sql:pipeline_success_ratio < 0.9
-    for: 10m
-    labels:
-      severity: warning
-    annotations:
-      summary: "Pipeline success ratio dropped"
-      description: "Success ratio < 90% over the past 10 minutes"
+    # Success ratio < 90% for 10 minutes
+    - alert: PipelineLowSuccessRatio
+      expr: nl2sql:pipeline_success_ratio < 0.9
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Pipeline success ratio dropped"
+        description: "Success ratio < 90% over the past 10 minutes"
 
-  # Alert: high generator p95 latency (>1.5s for 5 minutes)
-  - alert: GeneratorLatencyHigh
-    expr: nl2sql:stage_p95_ms{stage="generator"} > 1500
-    for: 5m
-    labels:
-      severity: warning
-    annotations:
-      summary: "Generator p95 latency high"
-      description: "Generator p95 > 1.5s for 5 minutes"
+    # Generator p95 latency > 1.5s for 5 minutes
+    - alert: GeneratorLatencyHigh
+      expr: nl2sql:stage_p95_ms{stage="generator"} > 1500
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: "Generator p95 latency high"
+        description: "Generator p95 > 1.5s for 5 minutes"
 
-  # Alert: unusual spike in Safety blocks
-  - alert: SafetyBlocksSpike
-    expr: rate(safety_blocks_total[5m]) > 0.5
-    for: 5m
-    labels:
-      severity: info
-    annotations:
-      summary: "Unusual Safety block rate"
-      description: "Safety blocks rate > 0.5 per minute (check inputs or safety rules)"
+    # Safety blocks spike — per minute (not per second)
+    - alert: SafetyBlocksSpike
+      expr: rate(safety_blocks_total[5m]) * 60 > 0.5
+      for: 5m
+      labels:
+        severity: info
+      annotations:
+        summary: "Unusual Safety block rate"
+        description: "Safety blocks > 0.5 per minute (5m window)"

scripts/smoke_metrics.sh

@@ -12,16 +12,19 @@ for q in \
 do
   curl -s -X POST "$API/nl2sql" \
     -H 'Content-Type: application/json' \
+    -H 'X-API-Key: dev-key' \
     -d "{\"query\":\"$q\"}" >/dev/null || true
 done
 
 # Send queries that trigger safety and verifier checks
 curl -s -X POST "$API/nl2sql" \
   -H 'Content-Type: application/json' \
+  -H 'X-API-Key: dev-key' \
   -d '{"query":"DELETE FROM users;"}' >/dev/null || true
 
 curl -s -X POST "$API/nl2sql" \
   -H 'Content-Type: application/json' \
+  -H 'X-API-Key: dev-key' \
   -d '{"query":"SELECT COUNT(*), country FROM customers;"}' >/dev/null || true
 
 # Print a snapshot of key Prometheus metrics