"""``/eval/runs`` — list / detail / start / cancel eval runs. CONTRACT.md §6: ``POST /eval/runs`` returns 202 ``{jobId, sseUrl}``. Terminal ``done`` event carries ``runId``. All other endpoints return sync JSON. Auth: all four endpoints require an authenticated user (admin-only for POST/cancel; any user may read GET endpoints — matches the history router pattern where GET /history is user-tier and write actions are admin-tier). Stage 7 scope: only the 4 run endpoints below. The golden-item CRUD (GET/POST/PATCH/DELETE /eval/golden) is also Stage 7 but lives in a separate ticket and is NOT in this file. """ from __future__ import annotations import logging import uuid from typing import Literal from fastapi import APIRouter, Depends, Path, Query from src.api.deps import forbid_readers, require_admin from src.api.dto.common import JobStartResponse from src.api.dto.evaluation import ( EvalItem, EvalRunDetail, EvalRunListItem, EvalRunStatusEnum, EvalRunsResponse, EvalStartRequest, ) from src.api.errors import ApiError from src.api.jobs import runner, store from src.api.jobs.lifecycle import cancel_job from src.api.limits import enforce_paid_request from src.api.validation import assert_known_model from src.lib.auth.models import User from src.lib.eval.repo import ( cancel_eval_run, fetch_eval_run, list_eval_runs, update_eval_run_complete, ) # Side-effect import: registers "eval_run" handler with the job registry. from src.api.jobs.types import eval_run as _eval_run_job # noqa: F401 logger = logging.getLogger(__name__) router = APIRouter(prefix="/eval", tags=["evaluation"]) # ---------- Errors -------------------------------------------------------- class EvalRunNotFound(ApiError): """``GET /eval/runs/{id}`` for an unknown id. Maps to 404.""" code = "EVAL_RUN_NOT_FOUND" status_code = 404 def __init__(self, *, run_id: int) -> None: super().__init__( f"No eval run with id {run_id}.", details={"runId": run_id}, ) class EvalRunNotCancellable(ApiError): """Cancel attempted on a terminal run. Maps to 409.""" code = "EVAL_RUN_NOT_CANCELLABLE" status_code = 409 def __init__(self, *, run_id: int, status: str) -> None: super().__init__( f"Eval run {run_id} is already in terminal state '{status}'.", details={"runId": run_id, "currentStatus": status}, ) # ---------- Helpers ------------------------------------------------------- def _row_to_list_item(row) -> EvalRunListItem: """Adapt a :class:`EvalRunRow` to the list DTO.""" from datetime import datetime, timezone def _parse_dt(s: str | None): if not s: return None try: # SQLite stores datetime('now') as "YYYY-MM-DD HH:MM:SS" # (no timezone). We treat it as UTC. return datetime.fromisoformat(s).replace(tzinfo=timezone.utc) except (TypeError, ValueError): return None started = _parse_dt(row.started_at) if started is None: # Fallback: use epoch so the model validator doesn't reject None started = datetime(1970, 1, 1, tzinfo=timezone.utc) return EvalRunListItem( id=row.id, name=row.name, status=EvalRunStatusEnum(row.status), started_at=started, completed_at=_parse_dt(row.completed_at), metrics_summary=row.metrics_summary, total_cost_usd=row.total_cost_usd, item_count=row.item_count, ) def _row_to_detail(row) -> EvalRunDetail: """Adapt a :class:`EvalRunRow` to the detail DTO.""" list_item = _row_to_list_item(row) items: list[EvalItem] = [] for i, raw in enumerate(row.items): items.append( EvalItem( id=raw.get("id", i + 1), input=raw.get("input", {}), output=raw.get("output", {}), expected=raw.get("expected"), metrics={ k: float(v) for k, v in (raw.get("metrics") or {}).items() if isinstance(v, (int, float)) }, duration_ms=int(raw.get("duration_ms", 0)), cost_usd=float(raw.get("cost_usd", 0.0)), ) ) return EvalRunDetail( **list_item.model_dump(), items=items, config=row.config or {}, logs=row.logs or [], job_id=row.job_id, ) # ---------- Endpoints ----------------------------------------------------- @router.get("/runs", response_model=EvalRunsResponse) def list_runs( user: User = Depends(forbid_readers), limit: int = Query(default=50, ge=1, le=200), cursor: str | None = Query(default=None), status: Literal["running", "completed", "failed", "cancelled"] | None = Query( default=None ), ) -> EvalRunsResponse: """List eval runs, newest first. Query params: - ``limit``: page size (default 50, max 200). - ``cursor``: opaque string from ``next_cursor`` in a prior response. - ``status``: filter by lifecycle state. """ cursor_int: int | None = None if cursor is not None: try: cursor_int = int(cursor) except ValueError: cursor_int = None # Tenancy: non-admins only see their own eval runs (names/configs/logs/ # metrics are otherwise cross-user readable); admins see all. rows = list_eval_runs( status=status, username=None if user.is_admin else user.username, limit=limit + 1, cursor=cursor_int, ) has_more = len(rows) > limit page_rows = rows[:limit] next_cursor = str(page_rows[-1].id) if has_more else None return EvalRunsResponse( items=[_row_to_list_item(r) for r in page_rows], next_cursor=next_cursor, ) @router.get("/runs/{runId}", response_model=EvalRunDetail) def get_run( run_id: int = Path(..., alias="runId", ge=1), user: User = Depends(forbid_readers), ) -> EvalRunDetail: """Full detail view for one eval run. Returns the run summary, per-item results, config, and captured logs. 404 ``EVAL_RUN_NOT_FOUND`` for unknown id, and the same 404 for a non-owner (no cross-user disclosure; admins see all). """ row = fetch_eval_run(run_id) if row is None: raise EvalRunNotFound(run_id=run_id) if not user.is_admin and (getattr(row, "username", None) or "") != user.username: raise EvalRunNotFound(run_id=run_id) return _row_to_detail(row) @router.post("/runs", status_code=202, response_model=JobStartResponse) def start_run( payload: EvalStartRequest, user: User = Depends(require_admin), ) -> JobStartResponse: """Start a new eval run. Returns 202 ``{jobId, sseUrl}``. The SSE stream emits log/stage/progress/done events (CONTRACT.md §4-5). Terminal ``done`` event carries ``result.runId`` — the ``eval_runs.id`` of the new run. Admin-only: eval runs may call paid APIs (``stage8_router.answer``). The run's subject_key is ``eval:{name}`` so two simultaneous runs for the same eval set serialize. """ enforce_paid_request(user) # daily spend ceiling + per-user rate limit # Constrain the eval model to the registry allow-list, same as /query # overrides — an unbounded value would let an admin pin every eval item to # the priciest model. "default"/None pass through to the configured model. assert_known_model(payload.model, field="model") job_id = uuid.uuid4().hex params = payload.model_dump() params["_username"] = user.username store.insert_job( id=job_id, type="eval_run", params_json=params, username=user.username, ) runner.enqueue( job_id, type_="eval_run", params=params, subject_key=f"eval:{payload.name}", ) return JobStartResponse(job_id=job_id, sse_url=f"/jobs/{job_id}/events") @router.post("/runs/{runId}/cancel", status_code=200) def cancel_run( run_id: int = Path(..., alias="runId", ge=1), _: User = Depends(require_admin), ) -> dict: """Cancel a running eval run. Returns 200 with updated status. If the run is already in a terminal state (completed / failed / cancelled), returns 409 ``EVAL_RUN_NOT_CANCELLABLE``. Implementation: cancels the underlying job via the shared :func:`src.api.jobs.lifecycle.cancel_job` — the SAME path as ``POST /jobs/{jobId}/cancel`` — so the cooperative-cancellation registry is set (the eval handler's paid-provider boundary stops issuing new paid calls), then flips the ``eval_runs`` row status to ``cancelled``. """ row = fetch_eval_run(run_id) if row is None: raise EvalRunNotFound(run_id=run_id) if row.status in ("completed", "failed", "cancelled"): raise EvalRunNotCancellable(run_id=run_id, status=row.status) # Cancel the underlying job through the one true cancel path. This is the # fix for the drift Codex flagged: the previous hand-rolled copy never set # the cancellation registry, so a running eval kept making paid calls. if row.job_id: cancel_job(row.job_id, reason="canceled by admin via /eval/runs/{runId}/cancel") # Flip the eval_runs row. cancel_eval_run(run_id) return {"runId": run_id, "status": "cancelled"}