feat(metrics): integrate Prometheus; expose /metrics; instrument safety & verifier

app/main.py

@@ -1,14 +1,10 @@
 import os
 import time
 from fastapi import FastAPI, Request, Response, HTTPException
-from fastapi.responses import PlainTextResponse
-from prometheus_client import (
-    Counter,
-    Histogram,
-    CollectorRegistry,
-    generate_latest,
-    CONTENT_TYPE_LATEST,
-)
+from fastapi.responses import PlainTextResponse, RedirectResponse
+from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST
+from nl2sql.prom import REGISTRY
+from app.routers import dev
 
 try:
     from dotenv import load_dotenv
@@ -19,7 +15,7 @@ except Exception:
 
 from app.routers import nl2sql
 
-# ---- Optionally restore uploaded DB map ----
+# ---- Optional DB map restore ----
 try:
     from app.routers.nl2sql import _load_db_map
 
@@ -27,17 +23,24 @@ try:
 except Exception as e:
     print(f"⚠️ DB map not restored: {e}")
 
-application: FastAPI = FastAPI(
+# ----------------------------------------------------------------------------
+#  App definition
+# ----------------------------------------------------------------------------
+application = FastAPI(
     title="NL2SQL Copilot Prototype",
     version=os.getenv("APP_VERSION", "0.1.0"),
     description="Convert natural language to safe & verified SQL",
 )
 
-application.include_router(nl2sql.router, prefix="/api/v1")  # e.g. /api/v1/nl2sql
-application.include_router(nl2sql.router)  # e.g. /nl2sql
+# Register only versioned API
+application.include_router(nl2sql.router, prefix="/api/v1")
 
-# ---- Prometheus metrics ----
-REGISTRY = CollectorRegistry()
+# Register Dev-only routes (only when APP_ENV=dev)
+if os.getenv("APP_ENV", "dev").lower() == "dev":
+    application.include_router(dev.router, prefix="/api/v1")
+# ----------------------------------------------------------------------------
+#  Prometheus Metrics Middleware
+# ----------------------------------------------------------------------------
 REQUEST_COUNT = Counter(
     "http_requests_total",
     "Total HTTP requests",
@@ -46,7 +49,7 @@ REQUEST_COUNT = Counter(
 )
 REQUEST_LATENCY = Histogram(
     "http_request_latency_seconds",
-    "Request latency",
+    "Request latency (seconds)",
     ["path", "method"],
     registry=REGISTRY,
 )
@@ -58,23 +61,26 @@ async def metrics_middleware(request: Request, call_next):
     response: Response = await call_next(request)
     elapsed = time.perf_counter() - start
     route = request.scope.get("route")
-    path = route.path if route else request.url.path
+    path = getattr(route, "path", None) or request.url.path
+    name = getattr(route, "name", None) or path
+
     REQUEST_COUNT.labels(
-        path=path, method=request.method, status_code=str(response.status_code)
+        path=name,
+        method=request.method,
+        status_code=str(getattr(response, "status_code", 500)),
     ).inc()
-    REQUEST_LATENCY.labels(path=path, method=request.method).observe(elapsed)
+    REQUEST_LATENCY.labels(path=name, method=request.method).observe(elapsed)
     return response
 
 
-# --- Liveness ---
+# ----------------------------------------------------------------------------
+#  System Endpoints
+# ----------------------------------------------------------------------------
 @application.get("/healthz", response_class=PlainTextResponse, tags=["system"])
 def healthz() -> str:
     return "ok"
 
 
-# --- Readiness ---
-
-
 @application.get("/readyz", response_class=PlainTextResponse, tags=["system"])
 def readyz() -> str:
     mode = os.getenv("DB_MODE", "sqlite").lower()
@@ -82,19 +88,17 @@ def readyz() -> str:
         if mode == "postgres":
             from adapters.db.postgres_adapter import PostgresAdapter
 
-            dsn = os.environ["POSTGRES_DSN"]
-            pg = PostgresAdapter(dsn)
-            ping = getattr(pg, "ping", None)
-            if callable(ping):
-                ping()
+            pg = PostgresAdapter(os.environ["POSTGRES_DSN"])
+            ping_fn = getattr(pg, "ping", None)
+            if callable(ping_fn):
+                ping_fn()
         else:
             from adapters.db.sqlite_adapter import SQLiteAdapter
 
-            db_path = os.getenv("SQLITE_DB_PATH", "data/chinook.db")
-            sq = SQLiteAdapter(db_path)
-            ping = getattr(sq, "ping", None)
-            if callable(ping):
-                ping()
+            sq = SQLiteAdapter(os.getenv("SQLITE_DB_PATH", "data/chinook.db"))
+            ping_fn = getattr(sq, "ping", None)
+            if callable(ping_fn):
+                ping_fn()
         return "ready"
     except Exception:
         raise HTTPException(status_code=503, detail="not ready")
@@ -116,6 +120,26 @@ def metrics():
     return Response(content=data, media_type=CONTENT_TYPE_LATEST)
 
 
-# Backward compatibility for tests & uvicorn targets
-app: FastAPI = application
+# ----------------------------------------------------------------------------
+#  Legacy Redirects (clean compatibility)
+# ----------------------------------------------------------------------------
+@application.api_route("/nl2sql", methods=["GET", "POST"])
+async def legacy_nl2sql_redirect(request: Request):
+    return RedirectResponse(url="/api/v1/nl2sql", status_code=307)
+
+
+@application.api_route(
+    "/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"]
+)
+async def legacy_catch_all(request: Request, path: str):
+    """Redirect old root-level endpoints to versioned API."""
+    if path.startswith("api/v1"):
+        return RedirectResponse(url=f"/{path}", status_code=307)
+    return RedirectResponse(url=f"/api/v1/{path}", status_code=307)
+
+
+# ----------------------------------------------------------------------------
+#  Backward-compatible alias for uvicorn
+# ----------------------------------------------------------------------------
+app = application
 __all__ = ["application", "app"]

app/routers/dev.py

@@ -0,0 +1,65 @@
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+from nl2sql.safety import Safety
+from nl2sql.verifier import Verifier
+
+# pick adapter for verifier (SQLite default)
+from adapters.db.sqlite_adapter import SQLiteAdapter
+
+from dataclasses import is_dataclass, asdict
+from typing import Any
+
+
+def _is_dataclass_instance(x: Any) -> bool:
+    # True only for dataclass *instances* (not classes)
+    return is_dataclass(x) and not isinstance(x, type)
+
+
+def _to_dict(obj: Any) -> dict:
+    # Pydantic v2
+    if hasattr(obj, "model_dump"):
+        return obj.model_dump()  # type: ignore[no-any-return]
+    # Pydantic v1
+    if hasattr(obj, "dict"):
+        return obj.dict()  # type: ignore[no-any-return]
+    # Dataclass instance
+    if _is_dataclass_instance(obj):
+        return asdict(obj)  # type: ignore[arg-type]
+    # Plain object
+    if hasattr(obj, "__dict__"):
+        return {k: v for k, v in obj.__dict__.items() if not k.startswith("_")}
+    return {"value": str(obj)}
+
+
+router = APIRouter(prefix="/_dev", tags=["dev"])
+
+
+class SQLBody(BaseModel):
+    sql: str
+
+
+@router.post("/safety")
+def dev_safety_check(body: SQLBody):
+    """
+    Run the Safety stage directly on a raw SQL string.
+    Used for metrics validation (Prometheus counters).
+    """
+    s = Safety()
+    res = s.check(body.sql)
+    return _to_dict(res)
+
+
+@router.post("/verifier")
+def dev_verifier_check(body: SQLBody):
+    """
+    Run the Verifier stage directly on a raw SQL string
+    with a real adapter connection.
+    """
+    try:
+        adapter = SQLiteAdapter("data/chinook.db")
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Adapter init failed: {e}")
+
+    v = Verifier()
+    res = v.verify(body.sql, adapter=adapter)
+    return _to_dict(res)

nl2sql/metrics.py

@@ -0,0 +1,60 @@
+from prometheus_client import Counter, Histogram
+from nl2sql.prom import REGISTRY
+
+
+# -----------------------------------------------------------------------------
+#  Stage-level metrics
+# -----------------------------------------------------------------------------
+stage_duration_ms = Histogram(
+    "stage_duration_ms",
+    "Duration (ms) of each pipeline stage",
+    ["stage"],  # e.g. detector|planner|generator|safety|verifier
+    buckets=(1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000),
+    registry=REGISTRY,
+)
+
+# -----------------------------------------------------------------------------
+#  Safety stage metrics
+# -----------------------------------------------------------------------------
+safety_blocks_total = Counter(
+    "safety_blocks_total",
+    "Count of blocked SQL queries by safety checks",
+    [
+        "reason"
+    ],  # e.g. forbidden_keyword, multiple_statements, non_readonly, explain_not_allowed
+    registry=REGISTRY,
+)
+
+safety_checks_total = Counter(
+    "safety_checks_total",
+    "Total SQL queries checked by safety",
+    ["ok"],  # "true" or "false"
+    registry=REGISTRY,
+)
+
+# -----------------------------------------------------------------------------
+#  Verifier stage metrics
+# -----------------------------------------------------------------------------
+verifier_checks_total = Counter(
+    "verifier_checks_total",
+    "Count of verifier checks (success/failure)",
+    ["ok"],  # "true" | "false"
+    registry=REGISTRY,
+)
+
+verifier_failures_total = Counter(
+    "verifier_failures_total",
+    "Count of verifier failures by type",
+    ["reason"],  # e.g. parse_error, semantic_check_error, adapter_failure
+    registry=REGISTRY,
+)
+
+# -----------------------------------------------------------------------------
+#  Pipeline-level metrics
+# -----------------------------------------------------------------------------
+pipeline_runs_total = Counter(
+    "pipeline_runs_total",
+    "Total number of full pipeline runs",
+    ["status"],  # ok | error | ambiguous
+    registry=REGISTRY,
+)

nl2sql/prom.py

@@ -0,0 +1,3 @@
+from prometheus_client import CollectorRegistry
+
+REGISTRY = CollectorRegistry()

nl2sql/safety.py

@@ -7,6 +7,8 @@ from typing import List, Pattern
 import sqlglot
 
 from nl2sql.types import StageResult, StageTrace
+from nl2sql.metrics import safety_blocks_total, stage_duration_ms, safety_checks_total
+
 
 # ------------------------- Zero-width & basic regexes -------------------------
 
@@ -166,12 +168,16 @@ class Safety:
 
         # 0) nil / size guard
         if not sql or not sql.strip():
+            safety_blocks_total.labels(reason="empty_sql").inc()
+            safety_checks_total.labels(ok="false").inc()
             return StageResult(
                 ok=False,
                 error=["empty_sql"],
                 trace=StageTrace(stage=self.name, duration_ms=_ms(t0)),
             )
         if len(sql) > _MAX_SQL_LEN:
+            safety_blocks_total.labels(reason="sql_too_long").inc()
+            safety_checks_total.labels(ok="false").inc()
             return StageResult(
                 ok=False,
                 error=["sql_too_long"],
@@ -185,6 +191,8 @@ class Safety:
         semicolon_count = _count_statements_semicolon(body)
         glot_count = _count_statements_sqlglot(body)
         if semicolon_count != 1 or glot_count != 1:
+            safety_blocks_total.labels(reason="multiple_statements").inc()
+            safety_checks_total.labels(ok="false").inc()
             return StageResult(
                 ok=False,
                 error=["Multiple statements detected"],
@@ -203,6 +211,8 @@ class Safety:
         m = _FORBIDDEN.search(scan_body)
         if m:
             tok = m.group(0).strip().lower()
+            safety_blocks_total.labels(reason="forbidden_keyword").inc()
+            safety_checks_total.labels(ok="false").inc()
             return StageResult(
                 ok=False,
                 error=[f"Forbidden: {tok}"],
@@ -212,6 +222,8 @@ class Safety:
             m2 = rx.search(scan_body)
             if m2:
                 tok = m2.group(0).strip().lower()
+                safety_blocks_total.labels(reason="forbidden_keyword").inc()
+                safety_checks_total.labels(ok="false").inc()
                 return StageResult(
                     ok=False,
                     error=[f"Forbidden: {tok}"],
@@ -223,6 +235,8 @@ class Safety:
             trees = sqlglot.parse(body)
             root = trees[0]
         except Exception as e:
+            safety_blocks_total.labels(reason="parse_error").inc()
+            safety_checks_total.labels(ok="false").inc()
             return StageResult(
                 ok=False,
                 error=["parse_error"],
@@ -241,6 +255,8 @@ class Safety:
                 t2 = sqlglot.parse_one(remainder)
                 t2_type = type(t2).__name__.lower() if t2 else ""
                 if t2_type in {"select", "with"}:
+                    stage_duration_ms.labels("safety").observe(_ms(t0) / 1.0)
+                    safety_checks_total.labels(ok="true").inc()
                     return StageResult(
                         ok=True,
                         data={
@@ -259,6 +275,8 @@ class Safety:
         is_explain = root_type == "explain"
 
         if is_explain and not self.allow_explain:
+            safety_blocks_total.labels(reason="explain_not_allowed").inc()
+            safety_checks_total.labels(ok="false").inc()
             return StageResult(
                 ok=False,
                 error=["EXPLAIN not allowed"],
@@ -266,6 +284,8 @@ class Safety:
             )
 
         if not (is_select_like or (is_explain and self.allow_explain)):
+            safety_blocks_total.labels(reason="non_select").inc()
+            safety_checks_total.labels(ok="false").inc()
             return StageResult(
                 ok=False,
                 error=[f"Non-SELECT statement: {root_type}"],
@@ -273,6 +293,8 @@ class Safety:
             )
 
         # 5) success
+        stage_duration_ms.labels("safety").observe(_ms(t0) / 1.0)
+        safety_checks_total.labels(ok="true").inc()
         return StageResult(
             ok=True,
             data={

nl2sql/verifier.py

@@ -8,6 +8,11 @@ import sqlglot
 from sqlglot import expressions as exp
 
 from nl2sql.types import StageResult, StageTrace
+from nl2sql.metrics import (
+    verifier_checks_total,
+    stage_duration_ms,
+    verifier_failures_total,
+)
 
 
 def _ms(t0: float) -> int:
@@ -182,6 +187,8 @@ class Verifier:
             # Check for common sqlglot error indicators
             # When sqlglot can't parse properly, it often creates Command or Unknown nodes
             if tree_type in ("Command", "Unknown"):
+                verifier_checks_total.labels(ok="false").inc()
+                verifier_failures_total.labels(reason="parse_error").inc()
                 return StageResult(
                     ok=False,
                     error=["parse_error"],
@@ -190,6 +197,8 @@ class Verifier:
 
             # Also check if the tree has errors attribute (some versions of sqlglot)
             if hasattr(tree, "errors") and tree.errors:
+                verifier_checks_total.labels(ok="false").inc()
+                verifier_failures_total.labels(reason="parse_error").inc()
                 return StageResult(
                     ok=False,
                     error=["parse_error"],
@@ -207,6 +216,8 @@ class Verifier:
                     for kw in ["selct", "slect", "selet", "seelct"]
                 ):
                     # Common misspellings of SELECT
+                    verifier_checks_total.labels(ok="false").inc()
+                    verifier_failures_total.labels(reason="parse_error").inc()
                     return StageResult(
                         ok=False,
                         error=["parse_error"],
@@ -214,6 +225,8 @@ class Verifier:
                     )
 
         except Exception:
+            verifier_checks_total.labels(ok="false").inc()
+            verifier_failures_total.labels(reason="parse_error").inc()
             return StageResult(
                 ok=False,
                 error=["parse_error"],
@@ -248,9 +261,11 @@ class Verifier:
                     and any_nonagg_col
                     and not (has_group or has_window or is_distinct)
                 ):
+                    verifier_failures_total.labels(reason="semantic_error").inc()
                     issues.append("aggregation_without_group_by")
         except Exception as e:
             # Don't crash the verifier; surface a soft issue and let fallback run
+            verifier_failures_total.labels(reason="semantic_error").inc()
             issues.append(f"semantic_check_error:{e!s}")
 
         # 3) Fallback textual scan — only if AST didn't already flag
@@ -276,6 +291,9 @@ class Verifier:
                         select_list = m_sel.group("sel")
                         # a comma strongly suggests mixing aggregate and non-aggregate in projection
                         if "," in select_list:
+                            verifier_failures_total.labels(
+                                reason="agg_without_group_by"
+                            ).inc()
                             issues.append("aggregation_without_group_by")
             except Exception:
                 # ignore fallback errors
@@ -287,12 +305,16 @@ class Verifier:
             ok_val = self._extract_ok(exec_result)
             if ok_val is False:
                 err = self._extract_error(exec_result)
+                verifier_failures_total.labels(reason="preview_exec_error").inc()
                 issues.append(f"exec_error:{err}" if err else "exec_error")
         except Exception as e:
+            verifier_failures_total.labels(reason="preview_exec_error").inc()
             issues.append(f"exec_exception:{e!s}")
 
         # 5) Final decision — AFTER all checks (note: no early return before fallback)
         if issues:
+            verifier_checks_total.labels(ok="false").inc()
+            stage_duration_ms.labels("verifier").observe(_ms(t0) / 1.0)
             return StageResult(
                 ok=False,
                 error=issues,
@@ -301,6 +323,8 @@ class Verifier:
                 ),
             )
 
+        verifier_checks_total.labels(ok="true").inc()
+        stage_duration_ms.labels("verifier").observe(_ms(t0) / 1.0)
         return StageResult(
             ok=True,
             data={"verified": True},