src/api/dto/evaluation.py

"""DTOs for ``/eval/runs`` endpoints (Stage 7).

Spec: BACKEND_BUILD.md §7 (Evaluation section) + orchestrator brief.
All models inherit :class:`ApiModel` for camelCase wire keys (CONTRACT.md §1).

``EvalStartRequest`` is the POST body; ``EvalRunListItem`` / ``EvalRunsResponse``
back ``GET /eval/runs``; ``EvalItem`` + ``EvalRunDetail`` back
``GET /eval/runs/{id}``.

The ``status`` field uses a ``str`` enum per BACKEND_BUILD.md §8 so OpenAPI
codegen produces a TS union (not ``string``).
"""
from __future__ import annotations

from datetime import datetime
from enum import Enum
from typing import Any

from .common import ApiModel


class EvalRunStatusEnum(str, Enum):
    """Lifecycle states for an eval run.

    ``running`` while the job is in flight; ``completed`` on success;
    ``failed`` on error; ``cancelled`` on user cancel.
    """

    running = "running"
    completed = "completed"
    failed = "failed"
    cancelled = "cancelled"


# ---------- List item (GET /eval/runs) ------------------------------------
class EvalRunListItem(ApiModel):
    """One row in the ``GET /eval/runs`` list.

    ``metrics_summary`` is a dict of aggregate metric floats, e.g.
    ``{"precision": 0.87, "recall": 0.91}``. It is ``None`` while the run
    is in flight (scoring hasn't happened yet).
    """

    id: int
    name: str
    status: EvalRunStatusEnum
    started_at: datetime
    completed_at: datetime | None = None
    metrics_summary: dict[str, float] | None = None
    total_cost_usd: float
    item_count: int


class EvalRunsResponse(ApiModel):
    """Paginated wrapper for ``GET /eval/runs``.

    Cursor-based: ``next_cursor`` is the smallest ``id`` on the current page
    (or ``None`` when exhausted). The FE passes it back as ``?cursor=`` to
    fetch older runs.
    """

    items: list[EvalRunListItem]
    next_cursor: str | None = None


# ---------- Detail view (GET /eval/runs/{id}) -----------------------------
class EvalItem(ApiModel):
    """One evaluated item within a run.

    ``input`` is the eval input (query, expected, etc.); ``output`` is what
    the model actually produced; ``expected`` is the golden reference (may be
    ``None`` for retrieval-only items). ``metrics`` is a per-item dict of
    floats (e.g. ``{"book_recall": 1.0, "page_recall": null}``).
    """

    id: int
    input: dict[str, Any]
    output: dict[str, Any]
    expected: dict[str, Any] | None = None
    metrics: dict[str, float]
    duration_ms: int
    cost_usd: float


class EvalRunDetail(EvalRunListItem):
    """Full detail view — list item fields plus per-item results, config, logs.

    Inherits all fields from :class:`EvalRunListItem`; adds:
    - ``items``: the per-query result list.
    - ``config``: the :class:`EvalStartRequest` config dict used.
    - ``logs``: captured log lines from the run.
    - ``job_id``: the underlying job id (so the FE can attach to /jobs/{id}/events
      for live progress on still-running runs). Null once the job row is pruned.
    """

    items: list[EvalItem]
    config: dict[str, Any]
    logs: list[str]
    job_id: str | None = None


# ---------- Request bodies ------------------------------------------------
class EvalStartRequest(ApiModel):
    """Body for ``POST /eval/runs``.

    ``dataset_id`` identifies which golden dataset to load (e.g.
    ``"golden_set"`` → ``eval/golden_set.jsonl``). ``model`` is the model
    under test (used to set ``generation_model`` in the router). ``config``
    is a free-form dict for extra params (e.g. ``{"top_k": 20, "no_llm": false}``).
    """

    name: str
    dataset_id: str
    model: str
    config: dict[str, Any] = {}