F:\Aoun Grad Project\AwnAiService\AwnDeploy

app/api/api.py

@@ -1,5 +1,5 @@
 from fastapi import APIRouter, Depends
-from app.api import prediction, ocr, chat, voice, feedback, kb_admin, health
+from app.api import prediction, ocr, chat, voice, feedback, kb_admin, health, metrics
 from app.core.auth import verify_api_key
 
 # All routes under /api require X-API-Key header
@@ -14,3 +14,4 @@ api_router.include_router(feedback.router, prefix="/ai", tags=["ai"])
 api_router.include_router(kb_admin.router, prefix="/ai", tags=["ai"])
 
 public_router.include_router(health.router, prefix="/ai", tags=["ai"])
+public_router.include_router(metrics.router, prefix="/ai", tags=["metrics"])

app/core/observability.py

@@ -0,0 +1,157 @@
+"""
+Centralised Prometheus metrics and helpers.
+
+We keep all metric definitions in one place so providers, Redis, and the
+router can push to the same registry without circular imports.
+"""
+
+from typing import Optional, Union
+
+from fastapi import APIRouter
+from fastapi.responses import Response
+from prometheus_client import (
+    CONTENT_TYPE_LATEST,
+    Counter,
+    Gauge,
+    Histogram,
+    generate_latest,
+)
+
+# ─── Metric Definitions ──────────────────────────────────────────────────────
+
+# LLM call performance
+llm_latency_ms = Histogram(
+    "llm_latency_ms",
+    "End-to-end LLM latency in milliseconds",
+    ["provider", "model"],
+    buckets=[50, 100, 200, 400, 800, 1200, 1500, 2000, 3000, 5000, 8000, 12000],
+)
+
+llm_ttft_ms = Histogram(
+    "llm_ttft_ms",
+    "Time to first token (TTFT) in milliseconds",
+    ["provider", "model"],
+    buckets=[50, 100, 200, 400, 800, 1200, 1500, 2000, 3000, 5000, 8000, 12000],
+)
+
+llm_calls_total = Counter(
+    "llm_calls_total",
+    "Total LLM calls attempted per provider/model",
+    ["provider", "model"],
+)
+
+llm_errors_total = Counter(
+    "llm_errors_total",
+    "Total LLM errors per provider/model/error_type",
+    ["provider", "model", "error_type"],
+)
+
+# Circuit breaker and bulkhead health
+llm_breaker_open_state = Gauge(
+    "llm_breaker_open_state",
+    "Circuit breaker state (1=open, 0.5=half-open, 0=closed)",
+    ["provider"],
+)
+
+llm_bulkhead_in_use = Gauge(
+    "llm_bulkhead_in_use",
+    "Concurrent in-flight LLM calls per provider",
+    ["provider"],
+)
+
+llm_bulkhead_capacity = Gauge(
+    "llm_bulkhead_capacity",
+    "Configured bulkhead slots per provider",
+    ["provider"],
+)
+
+llm_bulkhead_skips_total = Counter(
+    "llm_bulkhead_skips_total",
+    "Requests skipped because bulkhead was saturated",
+    ["provider"],
+)
+
+# Redis health
+redis_rtt_ms = Histogram(
+    "redis_rtt_ms",
+    "Redis round-trip latency in milliseconds",
+    ["endpoint"],
+    buckets=[1, 2, 5, 10, 25, 50, 75, 100, 250, 500, 1000],
+)
+
+redis_errors_total = Counter(
+    "redis_errors_total",
+    "Redis errors grouped by endpoint and operation",
+    ["endpoint", "operation"],
+)
+
+
+# ─── Helper Functions ───────────────────────────────────────────────────────
+
+def record_llm_success(provider: str, model: str, latency_ms: float, ttft_ms: Optional[float] = None) -> None:
+    llm_calls_total.labels(provider=provider, model=model).inc()
+    llm_latency_ms.labels(provider=provider, model=model).observe(latency_ms)
+    if ttft_ms is not None:
+        llm_ttft_ms.labels(provider=provider, model=model).observe(ttft_ms)
+
+
+def record_llm_error(provider: str, model: str, error_type: str = "unknown") -> None:
+    llm_calls_total.labels(provider=provider, model=model).inc()
+    llm_errors_total.labels(provider=provider, model=model, error_type=error_type).inc()
+
+
+def set_bulkhead_capacity(provider: str, capacity: int) -> None:
+    llm_bulkhead_capacity.labels(provider=provider).set(capacity)
+
+
+def bulkhead_acquire(provider: str) -> None:
+    llm_bulkhead_in_use.labels(provider=provider).inc()
+
+
+def bulkhead_release(provider: str) -> None:
+    """Decrement the in-use bulkhead gauge.
+
+    Callers (the _provider_slot context-manager) are responsible for
+    ensuring acquire/release are balanced.  We simply decrement — Prometheus
+    Gauges support negative values internally and we avoid touching the
+    private _value attribute which is not part of the public API.
+    """
+    llm_bulkhead_in_use.labels(provider=provider).dec()
+
+
+def bulkhead_skip(provider: str) -> None:
+    llm_bulkhead_skips_total.labels(provider=provider).inc()
+
+
+def set_breaker_state(provider: str, state: Union[str, "CircuitState"]) -> None:
+    """
+    Accepts either CircuitState enum or its string value.
+    Closed=0, Half-open=0.5, Open=1 — matches alert rules.
+    """
+    state_val = state.value if hasattr(state, "value") else str(state)
+    gauge_val = 0.0
+    if state_val == "open":
+        gauge_val = 1.0
+    elif state_val == "half_open":
+        gauge_val = 0.5
+    llm_breaker_open_state.labels(provider=provider).set(gauge_val)
+
+
+def observe_redis_rtt(endpoint: str, rtt_ms: float) -> None:
+    redis_rtt_ms.labels(endpoint=endpoint).observe(rtt_ms)
+
+
+def record_redis_error(endpoint: str, operation: str) -> None:
+    redis_errors_total.labels(endpoint=endpoint, operation=operation).inc()
+
+
+# ─── /metrics Endpoint (Prometheus scrape) ──────────────────────────────────
+
+metrics_router = APIRouter()
+
+
+@metrics_router.get("/metrics", include_in_schema=False)
+async def prometheus_metrics() -> Response:
+    """Expose Prometheus metrics for Grafana/Prometheus scrapers."""
+    return Response(generate_latest(), media_type=CONTENT_TYPE_LATEST)
+

app/core/redis_client.py

@@ -1,8 +1,12 @@
-import redis.asyncio as redis
+import asyncio
 import logging
 import os
+import time
 from typing import Optional, List
 
+import redis.asyncio as redis
+from app.core.observability import observe_redis_rtt, record_redis_error
+
 logger = logging.getLogger(__name__)
 
 class RedisManager:
@@ -13,6 +17,7 @@ class RedisManager:
         self.last_ping_error: Optional[str] = None
         self.last_ping_ts: float = 0.0
         self._ping_task: Optional["asyncio.Task"] = None
+        self.active_endpoint: str = ""
 
     def _parse_urls(self) -> List[str]:
         """
@@ -40,12 +45,14 @@ class RedisManager:
                 # Ping to verify
                 await self.redis.ping()
                 self.is_connected = True
+                self.active_endpoint = redis_url
                 await self._record_ping()
                 self._start_ping_loop()
                 logger.info(f"Connected to Redis at {redis_url}")
                 return
             except Exception as e:
                 last_error = e
+                record_redis_error(redis_url, "connect")
                 logger.warning(f"Redis connect failed at {redis_url}: {e}")
 
         logger.error(f"Failed to connect to any Redis endpoint ({urls}). System will degrade to local memory. Last error: {last_error}")
@@ -63,34 +70,44 @@ class RedisManager:
             logger.info("Disconnected from Redis")
 
     async def _record_ping(self):
-        """Measure RTT and store locally."""
+        """Measure RTT and record to Prometheus + local state."""
         if not self.redis:
             return
-        import time as _t
-        start = _t.perf_counter()
+        start = time.perf_counter()
         try:
             await self.redis.ping()
-            self.last_ping_ms = (_t.perf_counter() - start) * 1000
+            self.last_ping_ms = (time.perf_counter() - start) * 1000
             self.last_ping_error = None
-            self.last_ping_ts = _t.time()
+            self.last_ping_ts = time.time()
+            observe_redis_rtt(self.active_endpoint or "unknown", self.last_ping_ms)
         except Exception as e:
             self.last_ping_error = str(e)
-            self.last_ping_ts = _t.time()
+            self.last_ping_ts = time.time()
+            record_redis_error(self.active_endpoint or "unknown", "ping")
 
     def _start_ping_loop(self):
-        """Fire-and-forget periodic ping task for telemetry."""
+        """Fire-and-forget periodic ping task for RTT telemetry.
+
+        Uses a 30-second interval to align with the Redis client's own
+        ``health_check_interval`` and avoid unnecessary network chatter.
+        Handles ``CancelledError`` silently so graceful shutdown is clean.
+        """
         if self._ping_task:
             return
-        import asyncio
+
         async def _loop():
-            while True:
-                await asyncio.sleep(10)
-                await self._record_ping()
+            try:
+                while True:
+                    await asyncio.sleep(30)  # aligned with health_check_interval
+                    await self._record_ping()
+            except asyncio.CancelledError:
+                pass  # clean shutdown — do not re-raise
+
         try:
             loop = asyncio.get_running_loop()
             self._ping_task = loop.create_task(_loop())
         except RuntimeError:
-            # no running loop; skip
+            # No running event loop (sync context) — skip
             self._ping_task = None
 
 redis_client = RedisManager()

app/services/chat/api/circuit_breaker.py

@@ -35,6 +35,7 @@ from dataclasses import dataclass, field
 from enum import Enum
 from typing import Optional
 from app.core.redis_client import redis_client
+from app.core.observability import set_breaker_state
 
 logger = logging.getLogger(__name__)
 
@@ -76,6 +77,7 @@ class CircuitBreaker:
 
     def __post_init__(self):
         self._recovery_base = self.recovery_timeout
+        set_breaker_state(self.name, self.state)
 
     # ── Redis helpers ──────────────────────────────────────────────────────
     def _redis_key(self) -> str:
@@ -155,6 +157,7 @@ class CircuitBreaker:
             if elapsed >= self.recovery_timeout:
                 if not self._probe_in_flight:
                     self.state = CircuitState.HALF_OPEN
+                    set_breaker_state(self.name, self.state)
                     self._probe_in_flight = True
                     logger.info(
                         f"Circuit [{self.name}]: OPEN → HALF_OPEN "
@@ -184,6 +187,7 @@ class CircuitBreaker:
                 f"[CircuitBreaker] {self.name} {previous.value.upper()} → CLOSED "
                 f"(recovered, recovery_timeout reset to {self._recovery_base}s)"
             )
+        set_breaker_state(self.name, self.state)
         self._schedule(self._push_state())
 
     def record_failure(self, recovery_override: Optional[int] = None):
@@ -216,11 +220,14 @@ class CircuitBreaker:
                     self.MAX_RECOVERY_TIMEOUT
                 )
             self.state = CircuitState.OPEN
+            set_breaker_state(self.name, self.state)
             logger.warning(
                 f"[CircuitBreaker] {self.name} → OPEN "
                 f"(failures={self.failure_count}/{self.failure_threshold}, "
                 f"recovery_in={self.recovery_timeout}s, cycle={self._open_cycles})"
             )
+        else:
+            set_breaker_state(self.name, self.state)
         self._schedule(self._push_state())
 
     def seconds_until_retry(self) -> int:

app/services/chat/api/llm_router.py

@@ -20,10 +20,32 @@ from app.services.chat.api.circuit_breaker import circuit_registry
 from app.services.chat.api.model_registry import model_registry
 from app.core.redis_client import redis_client
 from app.core.telemetry import telemetry
+from app.core.observability import (
+    record_llm_error,
+    record_llm_success,
+    set_bulkhead_capacity,
+    bulkhead_acquire,
+    bulkhead_release,
+    bulkhead_skip,
+)
 from contextlib import asynccontextmanager
 
 logger = logging.getLogger(__name__)
 
+
+def _get_model_name(client) -> str:
+    """Extract the first active model name from a provider client.
+
+    Centralises the repetitive ``client._get_active_models()[0] if ...``
+    pattern used throughout the router, with a safe fallback of ``"unknown"``.
+    """
+    try:
+        models = client._get_active_models() if hasattr(client, "_get_active_models") else []
+        return models[0] if models else "unknown"
+    except Exception:
+        return "unknown"
+
+
 # Graceful degradation message shown to users when all providers are unavailable
 _DEGRADATION_MSG_AR = (
     "عذراً، جميع خدمات الذكاء الاصطناعي غير متوفرة حالياً بسبب ضغط مرتفع. "
@@ -230,6 +252,7 @@ class LLMRouter:
             for name, _ in self.providers:
                 self.provider_states[name] = ProviderState()
                 budget = self._concurrency_budget.get(name)
+                set_bulkhead_capacity(name, budget if budget else 0)
                 if budget and budget > 0:
                     self._semaphores[name] = asyncio.Semaphore(budget)
                     logger.info(f"  Bulkhead [{name}] initialised — {budget} concurrent slots")
@@ -266,6 +289,7 @@ class LLMRouter:
         if sem.locked():
             # _value == 0: every slot is in use
             self._bulkhead_skips[name] += 1
+            bulkhead_skip(name)
             logger.debug(
                 f"Bulkhead [{name}] saturated — 0/{budget} slots free "
                 f"(total skips: {self._bulkhead_skips[name]})"
@@ -276,11 +300,13 @@ class LLMRouter:
         # Capacity is available; acquire() resolves immediately (no await needed
         # for a semaphore with _value > 0, but we keep await for correctness)
         await sem.acquire()
+        bulkhead_acquire(name)
         logger.debug(f"Bulkhead [{name}] acquired — {sem._value}/{budget} slots remaining")
         try:
             yield True
         finally:
             sem.release()
+            bulkhead_release(name)
 
     async def get_active_requests(self) -> int:
         """
@@ -469,8 +495,9 @@ class LLMRouter:
                                 
                                 cb.record_success()
                                 self.provider_states[name].record_success(latency)
-                                model_name = client._get_active_models()[0] if hasattr(client, "_get_active_models") and client._get_active_models() else "unknown"
+                                model_name = _get_model_name(client)
                                 telemetry.record_success(name, model_name, latency)
+                                record_llm_success(name, model_name, latency)
                                 
                                 self._log_routing_decision(request_id, name, "success", latency)
                                 span.set_attribute("llm.latency_ms", latency)
@@ -478,8 +505,10 @@ class LLMRouter:
                                 return result
                             except Exception as e:
                                 last_error = e
-                                model_name = client._get_active_models()[0] if hasattr(client, "_get_active_models") and client._get_active_models() else "unknown"
+                                model_name = _get_model_name(client)
                                 telemetry.record_error(name, model_name)
+                                error_label = e.error_type.value if isinstance(e, LLMProviderError) else type(e).__name__
+                                record_llm_error(name, model_name, error_label)
                                 self._handle_provider_error(name, client, e, cb, request_id)
                                 span.record_exception(e)
                                 span.set_status(Status(StatusCode.ERROR))
@@ -503,10 +532,15 @@ class LLMRouter:
                         latency = (time.monotonic() - start) * 1000
                         bypass_cb.record_success()
                         self.provider_states[bypass_name].record_success(latency)
+                        model_name = _get_model_name(bypass_client)
+                        record_llm_success(bypass_name, model_name, latency)
                         self._log_routing_decision(request_id, bypass_name, "success_emergency_bypass", latency)
                         return result
                     except Exception as e:
                         last_error = e
+                        model_name = _get_model_name(bypass_client)
+                        error_label = e.error_type.value if isinstance(e, LLMProviderError) else type(e).__name__
+                        record_llm_error(bypass_name, model_name, error_label)
                         self._handle_provider_error(bypass_name, bypass_client, e, bypass_cb, request_id)
 
                 # ATTEMPT 2: Adaptive Non-blocking fast retry
@@ -549,12 +583,16 @@ class LLMRouter:
                                     
                                     cb.record_success()
                                     self.provider_states[name].record_success(latency)
-                                    
+                                    model_name = client._get_active_models()[0] if hasattr(client, "_get_active_models") and client._get_active_models() else "unknown"
+                                    record_llm_success(name, model_name, latency)
                                     self._log_routing_decision(request_id, name, "success_after_wait", latency)
                                     span.set_attribute("llm.latency_ms", latency)
                                     return result
                                 except Exception as e:
                                     last_error = e
+                                    model_name = client._get_active_models()[0] if hasattr(client, "_get_active_models") and client._get_active_models() else "unknown"
+                                    error_label = e.error_type.value if isinstance(e, LLMProviderError) else type(e).__name__
+                                    record_llm_error(name, model_name, error_label)
                                     self._handle_provider_error(name, client, e, cb, request_id)
                                     span.record_exception(e)
                                     span.set_status(Status(StatusCode.ERROR))
@@ -664,6 +702,8 @@ class LLMRouter:
                                 except Exception as gen_err:
                                     logger.exception(f"[ATTEMPT 1]   ❌ FAILED to create generator for {name}: {type(gen_err).__name__}: {gen_err}")
                                     last_error = gen_err
+                                    model_name = _get_model_name(client)
+                                    record_llm_error(name, model_name, type(gen_err).__name__)
                                     raise
                                 
                                 first = True
@@ -674,8 +714,9 @@ class LLMRouter:
                                         logger.info(f"[ATTEMPT 1]   ✓✓✓ SUCCESS! First chunk (TTFT) from {name} in {latency:.1f}ms")
                                         cb.record_success()
                                         self.provider_states[name].record_success(latency, ttft_ms=latency)
-                                        model_name = client._get_active_models()[0] if hasattr(client, "_get_active_models") and client._get_active_models() else "unknown"
+                                        model_name = _get_model_name(client)
                                         telemetry.record_success(name, model_name, latency)
+                                        record_llm_success(name, model_name, latency, ttft_ms=latency)
                                         
                                         self._log_routing_decision(request_id, name, "stream_started", latency)
                                         span.set_attribute("llm.latency_ms", latency)
@@ -688,8 +729,10 @@ class LLMRouter:
                             except Exception as e:
                                 last_error = e
                                 logger.exception(f"[ATTEMPT 1]   ❌ FAILED {name}: {type(e).__name__}: {e}")
-                                model_name = client._get_active_models()[0] if hasattr(client, "_get_active_models") and client._get_active_models() else "unknown"
+                                model_name = _get_model_name(client)
                                 telemetry.record_error(name, model_name)
+                                error_label = e.error_type.value if isinstance(e, LLMProviderError) else type(e).__name__
+                                record_llm_error(name, model_name, error_label)
                                 self._handle_provider_error(name, client, e, cb, request_id)
                                 span.record_exception(e)
                                 span.set_status(Status(StatusCode.ERROR))
@@ -726,12 +769,17 @@ class LLMRouter:
                                 logger.info(f"[EMERGENCY BYPASS] ✓ First chunk (TTFT) from {bypass_name} in {latency:.1f}ms")
                                 bypass_cb.record_success()
                                 self.provider_states[bypass_name].record_success(latency, ttft_ms=latency)
+                                model_name = _get_model_name(bypass_client)
+                                record_llm_success(bypass_name, model_name, latency, ttft_ms=latency)
                             yield chunk
                         logger.info(f"[EMERGENCY BYPASS] ✓✓✓ Stream completed via {bypass_name}")
                         return
                     except Exception as e:
                         last_error = e
                         logger.exception(f"[EMERGENCY BYPASS] ❌ {bypass_name} also failed: {e}")
+                        model_name = _get_model_name(bypass_client)
+                        error_label = e.error_type.value if isinstance(e, LLMProviderError) else type(e).__name__
+                        record_llm_error(bypass_name, model_name, error_label)
                         self._handle_provider_error(bypass_name, bypass_client, e, bypass_cb, request_id)
 
                 # ATTEMPT 2: Adaptive Non-blocking fast retry
@@ -791,6 +839,8 @@ class LLMRouter:
                                     except Exception as gen_err:
                                         logger.exception(f"[ATTEMPT 2]   ❌ Generator failed for {name}: {gen_err}")
                                         last_error = gen_err
+                                        model_name = _get_model_name(client)
+                                        record_llm_error(name, model_name, type(gen_err).__name__)
                                         raise
                                     
                                     first = True
@@ -801,6 +851,8 @@ class LLMRouter:
                                             logger.info(f"[ATTEMPT 2]   ✓✓✓ SUCCESS! First chunk (TTFT) from {name} in {latency:.1f}ms")
                                             cb.record_success()
                                             self.provider_states[name].record_success(latency, ttft_ms=latency)
+                                            model_name = _get_model_name(client)
+                                            record_llm_success(name, model_name, latency, ttft_ms=latency)
                                             self._log_routing_decision(request_id, name, "stream_started_after_wait", latency)
                                             span.set_attribute("llm.latency_ms", latency)
                                             span.set_attribute("llm.ttft_ms", latency)
@@ -811,6 +863,9 @@ class LLMRouter:
                                 except Exception as e:
                                     last_error = e
                                     logger.exception(f"[ATTEMPT 2]   ❌ FAILED {name}: {type(e).__name__}: {e}")
+                                    model_name = _get_model_name(client)
+                                    error_label = e.error_type.value if isinstance(e, LLMProviderError) else type(e).__name__
+                                    record_llm_error(name, model_name, error_label)
                                     self._handle_provider_error(name, client, e, cb, request_id)
                                     span.record_exception(e)
                                     span.set_status(Status(StatusCode.ERROR))

main.py

@@ -7,12 +7,15 @@ import time
 from fastapi import FastAPI, Request
 from fastapi.responses import RedirectResponse, JSONResponse
 from fastapi.middleware.gzip import GZipMiddleware
+from prometheus_fastapi_instrumentator import Instrumentator
+from prometheus_client import REGISTRY
 from contextlib import asynccontextmanager
 from fastapi.middleware.cors import CORSMiddleware
 from app.core.config import settings
 from app.api.api import api_router
 from app.core.redis_client import redis_client
 from app.core.otel import init_otel
+from app.core.observability import metrics_router  # exposes /metrics with all custom + instrumentator metrics
 
 # ─── Logging ──────────────────────────────────────────────────────────────────
 
@@ -122,6 +125,7 @@ def create_application() -> FastAPI:
     from app.api.api import public_router
     application.include_router(api_router, prefix="/api")
     application.include_router(public_router, prefix="/api")
+    application.include_router(metrics_router)  # /metrics for Prometheus/Grafana
 
     @application.get("/", include_in_schema=False)
     async def root():
@@ -169,6 +173,16 @@ def create_application() -> FastAPI:
         )
         return response
 
+    # Prometheus HTTP instrumentation (path-level latency, throughput, errors).
+    # We call instrument() to register the collection middleware but do NOT call
+    # expose() — our custom metrics_router at /metrics already calls
+    # generate_latest() which reads from the default registry, so instrumentator
+    # metrics (http_request_duration_seconds, etc.) are included automatically.
+    Instrumentator(
+        should_group_status_codes=True,
+        excluded_handlers={"/metrics", "/health", "/"},
+    ).instrument(application)
+
     return application
 
 

monitoring/alert_rules.yml

@@ -55,3 +55,15 @@ groups:
         annotations:
           summary: "p95 latency above SLO"
           description: "p95 latency >1.5s for provider {{ $labels.provider }} for 10m."
+
+      - alert: AllProvidersDown
+        expr: count(llm_breaker_open_state == 1) >= count(llm_breaker_open_state)
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "ALL LLM providers are down"
+          description: >
+            Every registered LLM provider circuit breaker is OPEN.
+            The service is fully degraded and returning cached/fallback responses only.
+            Investigate upstream API failures immediately.

monitoring/grafana_dashboard.json

@@ -1,108 +1,427 @@
 {
-  "title": "Awn AI Routing — Service Overview",
-  "tags": ["awn", "llm", "routing"],
+  "id": null,
+  "uid": "awn-ai-ops-v2",
+  "title": "Awn AI Service — Operational Dashboard",
+  "tags": ["awn", "llm", "observability"],
   "timezone": "browser",
   "schemaVersion": 38,
+  "refresh": "30s",
+  "time": { "from": "now-24h", "to": "now" },
+  "templating": {
+    "list": [
+      {
+        "type": "datasource",
+        "name": "datasource",
+        "label": "Prometheus",
+        "pluginId": "prometheus",
+        "refresh": 1
+      },
+      {
+        "type": "query",
+        "name": "provider",
+        "label": "Provider",
+        "datasource": { "type": "prometheus", "uid": "${datasource}" },
+        "query": "label_values(llm_calls_total, provider)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true
+      },
+      {
+        "type": "query",
+        "name": "model",
+        "label": "Model",
+        "datasource": { "type": "prometheus", "uid": "${datasource}" },
+        "query": "label_values(llm_calls_total, model)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true
+      }
+    ]
+  },
   "panels": [
     {
+      "id": 1,
       "type": "row",
-      "title": "Latency",
-      "panels": [
-        {
-          "type": "timeseries",
-          "title": "LLM Latency p95",
-          "targets": [
-            { "expr": "histogram_quantile(0.95, sum by (le,provider,model) (rate(llm_latency_ms_bucket[5m])))", "legendFormat": "{{provider}}/{{model}}" }
-          ]
-        },
+      "title": "🌐 Traffic & HTTP",
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+      "collapsed": false
+    },
+    {
+      "id": 2,
+      "type": "timeseries",
+      "title": "Requests per second",
+      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" } }
+      },
+      "targets": [
         {
-          "type": "timeseries",
-          "title": "LLM Latency p99",
-          "targets": [
-            { "expr": "histogram_quantile(0.99, sum by (le,provider,model) (rate(llm_latency_ms_bucket[5m])))", "legendFormat": "{{provider}}/{{model}}" }
-          ]
+          "expr": "sum(rate(http_requests_total{handler!=\"/metrics\"}[5m])) by (method)",
+          "legendFormat": "{{method}}",
+          "refId": "A"
         }
       ]
     },
     {
+      "id": 3,
+      "type": "timeseries",
+      "title": "HTTP 5xx error ratio",
+      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "color": { "mode": "fixed", "fixedColor": "red" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.01 },
+              { "color": "red", "value": 0.05 }
+            ]
+          }
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(http_requests_total{status=~\"5..\",handler!=\"/metrics\"}[5m])) / sum(rate(http_requests_total{handler!=\"/metrics\"}[5m]))",
+          "legendFormat": "5xx ratio",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 4,
+      "type": "timeseries",
+      "title": "HTTP latency p95 (by route)",
+      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": { "unit": "s" }
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, sum by (le,handler) (rate(http_request_duration_seconds_bucket{handler!=\"/metrics\"}[5m])))",
+          "legendFormat": "{{handler}}",
+          "refId": "A"
+        }
+      ]
+    },
+
+    {
+      "id": 10,
       "type": "row",
-      "title": "Reliability",
-      "panels": [
-        {
-          "type": "timeseries",
-          "title": "Error Rate",
-          "targets": [
-            { "expr": "sum by (provider,model) (rate(llm_errors_total[5m])) / sum by (provider,model) (rate(llm_calls_total[5m]))", "legendFormat": "{{provider}}/{{model}}" }
-          ]
-        },
+      "title": "🤖 LLM Performance",
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
+      "collapsed": false
+    },
+    {
+      "id": 11,
+      "type": "timeseries",
+      "title": "LLM end-to-end latency p95",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 800 },
+              { "color": "red", "value": 1500 }
+            ]
+          }
+        }
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, sum by (le,provider,model) (rate(llm_latency_ms_bucket[5m])))",
+          "legendFormat": "{{provider}}/{{model}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 12,
+      "type": "timeseries",
+      "title": "Time-to-First-Token p95 (TTFT)",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 500 },
+              { "color": "red", "value": 1200 }
+            ]
+          }
+        }
+      },
+      "targets": [
         {
-          "type": "timeseries",
-          "title": "Breaker State (OPEN=1)",
-          "targets": [
-            { "expr": "llm_breaker_open_state", "legendFormat": "{{provider}}" }
-          ]
+          "expr": "histogram_quantile(0.95, sum by (le,provider,model) (rate(llm_ttft_ms_bucket[5m])))",
+          "legendFormat": "{{provider}}/{{model}}",
+          "refId": "A"
         }
       ]
     },
     {
+      "id": 13,
+      "type": "timeseries",
+      "title": "LLM error rate by provider",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.02 },
+              { "color": "red", "value": 0.1 }
+            ]
+          }
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum by (provider) (rate(llm_errors_total[5m])) / sum by (provider) (rate(llm_calls_total[5m]))",
+          "legendFormat": "{{provider}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 14,
+      "type": "timeseries",
+      "title": "LLM call volume",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": { "unit": "reqps" }
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(llm_calls_total[5m])) by (provider,model)",
+          "legendFormat": "{{provider}}/{{model}}",
+          "refId": "A"
+        }
+      ]
+    },
+
+    {
+      "id": 20,
       "type": "row",
-      "title": "Capacity",
-      "panels": [
-        {
-          "type": "timeseries",
-          "title": "Bulkhead Utilization",
-          "targets": [
-            { "expr": "llm_bulkhead_in_use / llm_bulkhead_capacity", "legendFormat": "{{provider}}" }
-          ]
-        },
+      "title": "🛡 Resilience — Circuit Breakers & Bulkhead",
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 },
+      "collapsed": false
+    },
+    {
+      "id": 21,
+      "type": "timeseries",
+      "title": "Circuit breaker state (0=closed, 0.5=half-open, 1=open)",
+      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 27 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "max": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.4 },
+              { "color": "red", "value": 0.9 }
+            ]
+          }
+        }
+      },
+      "targets": [
+        {
+          "expr": "llm_breaker_open_state",
+          "legendFormat": "{{provider}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 22,
+      "type": "timeseries",
+      "title": "Bulkhead utilization (in_use / capacity)",
+      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 27 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.7 },
+              { "color": "red", "value": 0.9 }
+            ]
+          }
+        }
+      },
+      "targets": [
+        {
+          "expr": "llm_bulkhead_in_use / llm_bulkhead_capacity",
+          "legendFormat": "{{provider}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 23,
+      "type": "timeseries",
+      "title": "Bulkhead skips/s (shed requests)",
+      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 27 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": { "unit": "reqps", "color": { "mode": "fixed", "fixedColor": "orange" } }
+      },
+      "targets": [
         {
-          "type": "timeseries",
-          "title": "Bulkhead Skips",
-          "targets": [
-            { "expr": "rate(llm_bulkhead_skips_total[5m])", "legendFormat": "{{provider}}" }
-          ]
+          "expr": "rate(llm_bulkhead_skips_total[5m])",
+          "legendFormat": "{{provider}}",
+          "refId": "A"
         }
       ]
     },
+
     {
+      "id": 30,
       "type": "row",
-      "title": "Redis Health",
-      "panels": [
-        {
-          "type": "timeseries",
-          "title": "Redis RTT p95",
-          "targets": [
-            { "expr": "histogram_quantile(0.95, sum by (le) (rate(redis_rtt_ms_bucket[5m])))", "legendFormat": "rtt p95" }
-          ]
+      "title": "🔴 Redis Health",
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 35 },
+      "collapsed": false
+    },
+    {
+      "id": 31,
+      "type": "timeseries",
+      "title": "Redis RTT p95",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 20 },
+              { "color": "red", "value": 50 }
+            ]
+          }
+        }
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, sum by (le,endpoint) (rate(redis_rtt_ms_bucket[5m])))",
+          "legendFormat": "{{endpoint}} p95",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 32,
+      "type": "timeseries",
+      "title": "Redis errors/s",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": { "unit": "reqps", "color": { "mode": "fixed", "fixedColor": "red" } }
+      },
+      "targets": [
+        {
+          "expr": "rate(redis_errors_total[5m])",
+          "legendFormat": "{{endpoint}} / {{operation}}",
+          "refId": "A"
+        }
+      ]
+    },
+
+    {
+      "id": 40,
+      "type": "row",
+      "title": "📊 SLO Watch",
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 44 },
+      "collapsed": false
+    },
+    {
+      "id": 41,
+      "type": "timeseries",
+      "title": "p95 latency vs SLO (threshold = 1500ms)",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 45 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 800 },
+              { "color": "red", "value": 1500 }
+            ]
+          },
+          "custom": {
+            "thresholdsStyle": { "mode": "line+area" }
+          }
+        }
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, sum by (le,provider) (rate(llm_latency_ms_bucket[5m])))",
+          "legendFormat": "{{provider}} p95",
+          "refId": "A"
         },
         {
-          "type": "timeseries",
-          "title": "Redis Errors",
-          "targets": [
-            { "expr": "rate(redis_errors_total[5m])", "legendFormat": "errors" }
-          ]
+          "expr": "vector(1500)",
+          "legendFormat": "SLO limit (1500ms)",
+          "refId": "B"
         }
       ]
-    }
-  ],
-  "templating": {
-    "list": [
-      {
-        "type": "query",
-        "name": "provider",
-        "datasource": null,
-        "query": "label_values(llm_calls_total, provider)",
-        "refresh": 2
+    },
+    {
+      "id": 42,
+      "type": "timeseries",
+      "title": "Error budget burn rate (5m window)",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 45 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.02 },
+              { "color": "red", "value": 0.05 }
+            ]
+          }
+        }
       },
-      {
-        "type": "query",
-        "name": "model",
-        "datasource": null,
-        "query": "label_values(llm_calls_total, model)",
-        "refresh": 2
-      }
-    ]
-  },
-  "time": { "from": "now-24h", "to": "now" },
-  "refresh": "30s"
+      "targets": [
+        {
+          "expr": "sum by (provider) (rate(llm_errors_total[5m])) / sum by (provider) (rate(llm_calls_total[5m]))",
+          "legendFormat": "{{provider}} error rate",
+          "refId": "A"
+        },
+        {
+          "expr": "vector(0.02)",
+          "legendFormat": "Error budget (2%)",
+          "refId": "B"
+        }
+      ]
+    }
+  ]
 }

requirements.txt

@@ -3,6 +3,8 @@ uvicorn[standard]>=0.34.0,<1.0.0
 pydantic-settings>=2.0.0,<3.0.0
 python-multipart>=0.0.9
 python-dotenv>=1.0.0,<2.0.0
+prometheus-client>=0.20.0,<1.0.0
+prometheus-fastapi-instrumentator>=7.0.0,<8.0.0
 joblib>=1.4.0,<2.0.0
 scikit-learn==1.6.1
 xgboost>=2.1.0,<3.0.0