"""DTOs for ``/eval/runs`` endpoints (Stage 7). Spec: BACKEND_BUILD.md §7 (Evaluation section) + orchestrator brief. All models inherit :class:`ApiModel` for camelCase wire keys (CONTRACT.md §1). ``EvalStartRequest`` is the POST body; ``EvalRunListItem`` / ``EvalRunsResponse`` back ``GET /eval/runs``; ``EvalItem`` + ``EvalRunDetail`` back ``GET /eval/runs/{id}``. The ``status`` field uses a ``str`` enum per BACKEND_BUILD.md §8 so OpenAPI codegen produces a TS union (not ``string``). """ from __future__ import annotations from datetime import datetime from enum import Enum from typing import Any from .common import ApiModel class EvalRunStatusEnum(str, Enum): """Lifecycle states for an eval run. ``running`` while the job is in flight; ``completed`` on success; ``failed`` on error; ``cancelled`` on user cancel. """ running = "running" completed = "completed" failed = "failed" cancelled = "cancelled" # ---------- List item (GET /eval/runs) ------------------------------------ class EvalRunListItem(ApiModel): """One row in the ``GET /eval/runs`` list. ``metrics_summary`` is a dict of aggregate metric floats, e.g. ``{"precision": 0.87, "recall": 0.91}``. It is ``None`` while the run is in flight (scoring hasn't happened yet). """ id: int name: str status: EvalRunStatusEnum started_at: datetime completed_at: datetime | None = None metrics_summary: dict[str, float] | None = None total_cost_usd: float item_count: int class EvalRunsResponse(ApiModel): """Paginated wrapper for ``GET /eval/runs``. Cursor-based: ``next_cursor`` is the smallest ``id`` on the current page (or ``None`` when exhausted). The FE passes it back as ``?cursor=`` to fetch older runs. """ items: list[EvalRunListItem] next_cursor: str | None = None # ---------- Detail view (GET /eval/runs/{id}) ----------------------------- class EvalItem(ApiModel): """One evaluated item within a run. ``input`` is the eval input (query, expected, etc.); ``output`` is what the model actually produced; ``expected`` is the golden reference (may be ``None`` for retrieval-only items). ``metrics`` is a per-item dict of floats (e.g. ``{"book_recall": 1.0, "page_recall": null}``). """ id: int input: dict[str, Any] output: dict[str, Any] expected: dict[str, Any] | None = None metrics: dict[str, float] duration_ms: int cost_usd: float class EvalRunDetail(EvalRunListItem): """Full detail view — list item fields plus per-item results, config, logs. Inherits all fields from :class:`EvalRunListItem`; adds: - ``items``: the per-query result list. - ``config``: the :class:`EvalStartRequest` config dict used. - ``logs``: captured log lines from the run. - ``job_id``: the underlying job id (so the FE can attach to /jobs/{id}/events for live progress on still-running runs). Null once the job row is pruned. """ items: list[EvalItem] config: dict[str, Any] logs: list[str] job_id: str | None = None # ---------- Request bodies ------------------------------------------------ class EvalStartRequest(ApiModel): """Body for ``POST /eval/runs``. ``dataset_id`` identifies which golden dataset to load (e.g. ``"golden_set"`` → ``eval/golden_set.jsonl``). ``model`` is the model under test (used to set ``generation_model`` in the router). ``config`` is a free-form dict for extra params (e.g. ``{"top_k": 20, "no_llm": false}``). """ name: str dataset_id: str model: str config: dict[str, Any] = {}