feat(observability): add Prometheus-Grafana stack with auto-provisioning and docs

Makefile

@@ -109,12 +109,14 @@ clean: ## Remove Python caches
 clean-all: clean ## Remove build artifacts and coverage
 	rm -rf dist build .coverage *.egg-info
 
-# ---------- Metrics ----------
-.PHONY: prom-up prom-check smoke
+# ---------- Observability Stack ----------
+.PHONY: obs-up obs-down obs-logs prom-up prom-check smoke
 
+# Bring up Prometheus + Grafana via Docker Compose
 prom-up:
 	docker compose -f docker-compose.prom.yml up -d
 
+# Validate Prometheus configs (fallback to Docker if promtool is missing)
 prom-check:
 	@if command -v promtool >/dev/null 2>&1; then \
 		echo "🔍 Running promtool locally..."; \
@@ -127,5 +129,93 @@ prom-check:
 			promtool check config /etc/prometheus/prometheus.yml; \
 	fi
 
+# Generate sample traffic and print key metrics snapshot
 smoke:
 	./scripts/smoke_metrics.sh
+
+# Bring up the stack, wait until services are ready, then run smoke
+obs-up:
+	@set -e; \
+	\
+	# 1) Up the stack
+	$(MAKE) prom-up; \
+	\
+	# 2) Wait for Prometheus readiness
+	#    - Tries the /-/ready endpoint (preferred). Falls back to port check.
+	#    - Times out after ~90s.
+	echo "⏳ Waiting for Prometheus (http://localhost:9090) ..."; \
+	for i in $$(seq 1 30); do \
+		# Check readiness endpoint
+		if curl -fsS http://localhost:9090/-/ready >/dev/null 2>&1; then \
+			echo "✅ Prometheus is ready"; \
+			break; \
+		fi; \
+		# Fallback: check TCP port if /-/ready is not enabled
+		if nc -z localhost 9090 >/dev/null 2>&1; then \
+			echo "✅ Prometheus port is open (assuming ready)"; \
+			break; \
+		fi; \
+		sleep 3; \
+		if [ $$i -eq 30 ]; then \
+			echo "❌ Prometheus did not become ready in time"; \
+			exit 1; \
+		fi; \
+	done; \
+	\
+	# 3) Wait for Grafana login page
+	#    - Checks that /login returns HTTP 200/302.
+	echo "⏳ Waiting for Grafana (http://localhost:3000) ..."; \
+	for i in $$(seq 1 30); do \
+		code=$$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3000/login || true); \
+		if [ "$$code" = "200" ] || [ "$$code" = "302" ]; then \
+			echo "✅ Grafana is up"; \
+			break; \
+		fi; \
+		sleep 3; \
+		if [ $$i -eq 30 ]; then \
+			echo "❌ Grafana did not become ready in time"; \
+			exit 1; \
+		fi; \
+	done; \
+	\
+	# 4) Run smoke to populate metrics
+	echo "🚀 Running smoke traffic ..."; \
+	$(MAKE) smoke; \
+	echo "🎉 Observability stack is live. Open: Prometheus → http://localhost:9090 , Grafana → http://localhost:3000"
+    # 5) Auto-import Grafana dashboard
+	$(MAKE) grafana-import
+
+# Tear down the observability stack
+obs-down:
+	docker compose -f docker-compose.prom.yml down
+
+# Tail logs of both services
+obs-logs:
+	docker compose -f docker-compose.prom.yml logs -f
+
+# ---------- Grafana Auto Import ----------
+.PHONY: grafana-import
+
+# Import dashboard JSON into Grafana via HTTP API
+grafana-import:
+	@set -e; \
+	echo "⏳ Waiting for Grafana API to become ready..."; \
+	for i in $$(seq 1 30); do \
+		code=$$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3000/api/health || true); \
+		if [ "$$code" = "200" ]; then \
+			echo "✅ Grafana API is ready"; \
+			break; \
+		fi; \
+		sleep 3; \
+		if [ $$i -eq 30 ]; then \
+			echo "❌ Grafana API did not become ready in time"; \
+			exit 1; \
+		fi; \
+	done; \
+	\
+	echo "📦 Importing dashboard ..."; \
+	curl -s -X POST http://admin:admin@localhost:3000/api/dashboards/db \
+		-H "Content-Type: application/json" \
+		-d "{\"dashboard\": $$(cat prometheus/grafana_dashboard.json), \"overwrite\": true, \"folderId\": 0}" \
+		| jq -r '.status' || true; \
+	echo "🎉 Dashboard imported → http://localhost:3000/dashboards"

docker-compose.prom.yml

@@ -1,4 +1,5 @@
 version: "3.8"
+
 services:
   prometheus:
     image: prom/prometheus:v2.55.0
@@ -9,5 +10,32 @@ services:
       - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
       - ./prometheus/rules.yml:/etc/prometheus/rules.yml:ro
     ports:
-      - "9090:9090"
+      - "9090:9090"          # Prometheus UI
+    networks:
+      - observability
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: nl2sql-grafana
+    ports:
+      - "3000:3000"          # Grafana UI
+    depends_on:
+      - prometheus
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_USERS_ALLOW_SIGN_UP=false
+      - GF_AUTH_ANONYMOUS_ENABLED=false
+      # Optional hardening:
+      - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/etc/grafana/provisioning/dashboards/nl2sql.json
+    volumes:
+      # Provisioning (datasource + dashboard providers)
+      - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
+      - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
+
+      # The actual dashboard JSON (mounted into the same dashboards dir)
+      - ./prometheus/grafana_dashboard.json:/etc/grafana/provisioning/dashboards/nl2sql.json:ro
+    networks:
+      - observability
     restart: unless-stopped

docs/observability.md

@@ -0,0 +1,33 @@
+# Observability and Metrics
+
+This module adds full observability for the NL2SQL Copilot pipeline.
+
+## 📊 Metrics exposed
+
+| Metric | Type | Labels | Description |
+|--------|------|---------|--------------|
+| `stage_duration_ms` | histogram | `stage` | Duration per stage (detector, planner, generator, safety, executor, verifier) |
+| `pipeline_runs_total` | counter | `status` | Pipeline runs by outcome (`ok`, `error`, `ambiguous`) |
+| `safety_checks_total`, `safety_blocks_total` | counter | `reason` | Number of safety checks and blocked queries |
+| `verifier_checks_total`, `verifier_failures_total` | counter | `reason` | Number of verification passes and failures |
+
+---
+
+## ⚙️ Recording & Alerting Rules
+
+Defined in `prometheus/rules.yml`:
+
+- **`nl2sql:stage_p95_ms`** – 95th percentile latency per stage
+- **`nl2sql:pipeline_success_ratio`** – 5-minute success ratio
+- Alerts:
+  - `PipelineLowSuccessRatio` (<90% for 10m)
+  - `GeneratorLatencyHigh` (>1500 ms for 5m)
+  - `SafetyBlocksSpike` (>0.5/min)
+
+---
+
+## 🧪 Local Testing
+
+1. Start Prometheus
+   ```bash
+   make prom-up

grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,13 @@
+apiVersion: 1
+
+providers:
+  - name: "NL2SQL Dashboards"
+    orgId: 1
+    folder: ""
+    type: file
+    disableDeletion: true            # Prevent dashboard deletion from UI
+    editable: false
+    updateIntervalSeconds: 10        # How often Grafana scans for changes
+    options:
+      path: /etc/grafana/provisioning/dashboards
+      # Grafana will load any *.json from this directory

grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy          # Grafana will proxy queries
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+    jsonData:
+      httpMethod: GET      # Use GET for Prometheus

prometheus/grafana_dashboard.json

@@ -0,0 +1,68 @@
+{
+  "title": "NL2SQL Copilot - Observability",
+  "editable": true,
+  "panels": [
+    {
+      "type": "timeseries",
+      "title": "Stage p95 Latency (ms)",
+      "targets": [
+        {
+          "expr": "nl2sql:stage_p95_ms",
+          "legendFormat": "{{stage}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "milliseconds",
+          "decimals": 0
+        }
+      },
+      "id": 1
+    },
+    {
+      "type": "timeseries",
+      "title": "Pipeline Success Ratio",
+      "targets": [
+        {
+          "expr": "nl2sql:pipeline_success_ratio",
+          "legendFormat": "success ratio",
+          "refId": "B"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "max": 1,
+          "decimals": 2
+        }
+      },
+      "id": 2
+    },
+    {
+      "type": "timeseries",
+      "title": "Safety & Verifier Events",
+      "targets": [
+        {
+          "expr": "rate(safety_blocks_total[5m])",
+          "legendFormat": "safety blocks/min",
+          "refId": "C"
+        },
+        {
+          "expr": "rate(verifier_failures_total[5m])",
+          "legendFormat": "verifier failures/min",
+          "refId": "D"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "min": 0
+        }
+      },
+      "id": 3
+    }
+  ],
+  "schemaVersion": 38,
+  "version": 1,
+  "refresh": "30s"
+}