src/api/routers/evaluation.py

"""``/eval/runs`` — list / detail / start / cancel eval runs.

CONTRACT.md §6: ``POST /eval/runs`` returns 202 ``{jobId, sseUrl}``. Terminal
``done`` event carries ``runId``. All other endpoints return sync JSON.

Auth: all four endpoints require an authenticated user (admin-only for
POST/cancel; any user may read GET endpoints — matches the history router
pattern where GET /history is user-tier and write actions are admin-tier).

Stage 7 scope: only the 4 run endpoints below. The golden-item CRUD
(GET/POST/PATCH/DELETE /eval/golden) is also Stage 7 but lives in a separate
ticket and is NOT in this file.
"""
from __future__ import annotations

import logging
import uuid
from typing import Literal

from fastapi import APIRouter, Depends, Path, Query

from src.api.deps import forbid_readers, require_admin
from src.api.dto.common import JobStartResponse
from src.api.dto.evaluation import (
    EvalItem,
    EvalRunDetail,
    EvalRunListItem,
    EvalRunStatusEnum,
    EvalRunsResponse,
    EvalStartRequest,
)
from src.api.errors import ApiError
from src.api.jobs import runner, store
from src.api.jobs.lifecycle import cancel_job
from src.api.limits import enforce_paid_request
from src.api.validation import assert_known_model
from src.lib.auth.models import User
from src.lib.eval.repo import (
    cancel_eval_run,
    fetch_eval_run,
    list_eval_runs,
    update_eval_run_complete,
)

# Side-effect import: registers "eval_run" handler with the job registry.
from src.api.jobs.types import eval_run as _eval_run_job  # noqa: F401

logger = logging.getLogger(__name__)

router = APIRouter(prefix="/eval", tags=["evaluation"])


# ---------- Errors --------------------------------------------------------
class EvalRunNotFound(ApiError):
    """``GET /eval/runs/{id}`` for an unknown id. Maps to 404."""

    code = "EVAL_RUN_NOT_FOUND"
    status_code = 404

    def __init__(self, *, run_id: int) -> None:
        super().__init__(
            f"No eval run with id {run_id}.",
            details={"runId": run_id},
        )


class EvalRunNotCancellable(ApiError):
    """Cancel attempted on a terminal run. Maps to 409."""

    code = "EVAL_RUN_NOT_CANCELLABLE"
    status_code = 409

    def __init__(self, *, run_id: int, status: str) -> None:
        super().__init__(
            f"Eval run {run_id} is already in terminal state '{status}'.",
            details={"runId": run_id, "currentStatus": status},
        )


# ---------- Helpers -------------------------------------------------------
def _row_to_list_item(row) -> EvalRunListItem:
    """Adapt a :class:`EvalRunRow` to the list DTO."""
    from datetime import datetime, timezone

    def _parse_dt(s: str | None):
        if not s:
            return None
        try:
            # SQLite stores datetime('now') as "YYYY-MM-DD HH:MM:SS"
            # (no timezone). We treat it as UTC.
            return datetime.fromisoformat(s).replace(tzinfo=timezone.utc)
        except (TypeError, ValueError):
            return None

    started = _parse_dt(row.started_at)
    if started is None:
        # Fallback: use epoch so the model validator doesn't reject None
        started = datetime(1970, 1, 1, tzinfo=timezone.utc)

    return EvalRunListItem(
        id=row.id,
        name=row.name,
        status=EvalRunStatusEnum(row.status),
        started_at=started,
        completed_at=_parse_dt(row.completed_at),
        metrics_summary=row.metrics_summary,
        total_cost_usd=row.total_cost_usd,
        item_count=row.item_count,
    )


def _row_to_detail(row) -> EvalRunDetail:
    """Adapt a :class:`EvalRunRow` to the detail DTO."""
    list_item = _row_to_list_item(row)

    items: list[EvalItem] = []
    for i, raw in enumerate(row.items):
        items.append(
            EvalItem(
                id=raw.get("id", i + 1),
                input=raw.get("input", {}),
                output=raw.get("output", {}),
                expected=raw.get("expected"),
                metrics={
                    k: float(v)
                    for k, v in (raw.get("metrics") or {}).items()
                    if isinstance(v, (int, float))
                },
                duration_ms=int(raw.get("duration_ms", 0)),
                cost_usd=float(raw.get("cost_usd", 0.0)),
            )
        )

    return EvalRunDetail(
        **list_item.model_dump(),
        items=items,
        config=row.config or {},
        logs=row.logs or [],
        job_id=row.job_id,
    )


# ---------- Endpoints -----------------------------------------------------
@router.get("/runs", response_model=EvalRunsResponse)
def list_runs(
    user: User = Depends(forbid_readers),
    limit: int = Query(default=50, ge=1, le=200),
    cursor: str | None = Query(default=None),
    status: Literal["running", "completed", "failed", "cancelled"] | None = Query(
        default=None
    ),
) -> EvalRunsResponse:
    """List eval runs, newest first.

    Query params:
    - ``limit``: page size (default 50, max 200).
    - ``cursor``: opaque string from ``next_cursor`` in a prior response.
    - ``status``: filter by lifecycle state.
    """
    cursor_int: int | None = None
    if cursor is not None:
        try:
            cursor_int = int(cursor)
        except ValueError:
            cursor_int = None

    # Tenancy: non-admins only see their own eval runs (names/configs/logs/
    # metrics are otherwise cross-user readable); admins see all.
    rows = list_eval_runs(
        status=status,
        username=None if user.is_admin else user.username,
        limit=limit + 1,
        cursor=cursor_int,
    )

    has_more = len(rows) > limit
    page_rows = rows[:limit]
    next_cursor = str(page_rows[-1].id) if has_more else None

    return EvalRunsResponse(
        items=[_row_to_list_item(r) for r in page_rows],
        next_cursor=next_cursor,
    )


@router.get("/runs/{runId}", response_model=EvalRunDetail)
def get_run(
    run_id: int = Path(..., alias="runId", ge=1),
    user: User = Depends(forbid_readers),
) -> EvalRunDetail:
    """Full detail view for one eval run.

    Returns the run summary, per-item results, config, and captured logs.
    404 ``EVAL_RUN_NOT_FOUND`` for unknown id, and the same 404 for a
    non-owner (no cross-user disclosure; admins see all).
    """
    row = fetch_eval_run(run_id)
    if row is None:
        raise EvalRunNotFound(run_id=run_id)
    if not user.is_admin and (getattr(row, "username", None) or "") != user.username:
        raise EvalRunNotFound(run_id=run_id)
    return _row_to_detail(row)


@router.post("/runs", status_code=202, response_model=JobStartResponse)
def start_run(
    payload: EvalStartRequest,
    user: User = Depends(require_admin),
) -> JobStartResponse:
    """Start a new eval run.

    Returns 202 ``{jobId, sseUrl}``. The SSE stream emits log/stage/progress/done
    events (CONTRACT.md §4-5). Terminal ``done`` event carries
    ``result.runId`` — the ``eval_runs.id`` of the new run.

    Admin-only: eval runs may call paid APIs (``stage8_router.answer``).
    The run's subject_key is ``eval:{name}`` so two simultaneous runs for
    the same eval set serialize.
    """
    enforce_paid_request(user)  # daily spend ceiling + per-user rate limit
    # Constrain the eval model to the registry allow-list, same as /query
    # overrides — an unbounded value would let an admin pin every eval item to
    # the priciest model. "default"/None pass through to the configured model.
    assert_known_model(payload.model, field="model")
    job_id = uuid.uuid4().hex
    params = payload.model_dump()
    params["_username"] = user.username

    store.insert_job(
        id=job_id,
        type="eval_run",
        params_json=params,
        username=user.username,
    )
    runner.enqueue(
        job_id,
        type_="eval_run",
        params=params,
        subject_key=f"eval:{payload.name}",
    )
    return JobStartResponse(job_id=job_id, sse_url=f"/jobs/{job_id}/events")


@router.post("/runs/{runId}/cancel", status_code=200)
def cancel_run(
    run_id: int = Path(..., alias="runId", ge=1),
    _: User = Depends(require_admin),
) -> dict:
    """Cancel a running eval run.

    Returns 200 with updated status. If the run is already in a terminal state
    (completed / failed / cancelled), returns 409 ``EVAL_RUN_NOT_CANCELLABLE``.

    Implementation: cancels the underlying job via the shared
    :func:`src.api.jobs.lifecycle.cancel_job` — the SAME path as
    ``POST /jobs/{jobId}/cancel`` — so the cooperative-cancellation registry
    is set (the eval handler's paid-provider boundary stops issuing new paid
    calls), then flips the ``eval_runs`` row status to ``cancelled``.
    """
    row = fetch_eval_run(run_id)
    if row is None:
        raise EvalRunNotFound(run_id=run_id)

    if row.status in ("completed", "failed", "cancelled"):
        raise EvalRunNotCancellable(run_id=run_id, status=row.status)

    # Cancel the underlying job through the one true cancel path. This is the
    # fix for the drift Codex flagged: the previous hand-rolled copy never set
    # the cancellation registry, so a running eval kept making paid calls.
    if row.job_id:
        cancel_job(row.job_id, reason="canceled by admin via /eval/runs/{runId}/cancel")

    # Flip the eval_runs row.
    cancel_eval_run(run_id)
    return {"runId": run_id, "status": "cancelled"}