src/endopath/case_review.py

"""Assemble the case-review confirmation surface and its value-chat co-pilot (#98).

The confirmation surface (docs/prd.md sections 8.1, 8.3, 4.6; design-guideline section 9.3) shows the
scanned page on the left and the extracted record on the right, each value beside the region of the page
it came from. A reviewer confirms each value against its evidence. This is the reviewer-confirmation
pattern the crosswalk (#103) established and this screen inherits (clinical-review-interface skill): the
engine drafts and the reviewer corrects, identity comes from the session, every value is editable, a chat
sits on each value, every decision writes a ledger record, and attention follows field type rather than a
confidence number.

This module is the read side. It reads the stored per-case checklist (`schema.EndometrialChecklist`, the
confirmable unit today) and, per field:

- runs the B1 validation pass (#97, `validation.py`) to produce the per-field validation block the
  confirmation card renders, promoting a field from needs-review to confirmed-eligible;
- names the canonical concept the field answers, from the committed coverage bridge
  (`mapping.cap_checklist_coverage`, #95), so a card whose concept is derived renders its derivation;
- joins the field's recorded decision from the ledger (`storage.field_confirmations`, #103's pattern).

The validation block's shape checks run against the checklist field's own declared schema
(`fields.FieldSpec.value_schema`), the schema the value was extracted against, so a validly-stored value
carries no spurious failure. The concept the field maps to supplies the assertion type and, for a derived
concept, the named rule and its input concepts, so the derived card renders its derivation chip. A derived
concept whose primitive inputs are not separately stored on the checklist reports its derivation
`inputs_absent` with the reason named, and the field stays needs-review until a person confirms it: the
derivation is advisory here, never silently trusted (invariant 1). A CAP field the concept list does not
model (`mapping.UNMODELED_CAP_FIELDS`, empty in the MVP) validates against its checklist schema alone and
names that it maps to no canonical concept.

A value counts as data only after a person confirms it (CLAUDE.md invariant), so the promotion a
validation block earns is `confirmed_eligible`, not `confirmed`: the reviewer still signs it.
"""

from __future__ import annotations

import json
from datetime import datetime, timezone
from types import SimpleNamespace
from typing import Optional, Protocol, runtime_checkable

import anthropic
from sqlalchemy import Connection

from endopath import crosswalk, fields, mapping, storage
from endopath.llm_extraction import get_client
from endopath.schema import Case, ChecklistField, EvidenceSpan, FieldStatus
from endopath.validation import (
    ExtractedValue,
    FieldState,
    FieldValidation,
    Promotion,
    ValueShape,
    ValueType,
    validate_field,
)

# The right pane groups the checklist fields into collapsible bands (design-guideline section 9.3). The
# order within a section follows the CAP checklist. A field named here that the checklist does not carry
# is skipped, and any field the checklist carries but no section names lands in a trailing "OTHER" band,
# so a field added to `fields.FIELDS` is never silently dropped from the surface.
SECTIONS: tuple[tuple[str, tuple[str, ...]], ...] = (
    ("TUMOR", ("histologic_type", "histologic_grade", "lymphovascular_space_invasion")),
    ("ANATOMICAL INVOLVEMENT", ("myometrial_invasion_percent", "cervical_stromal_invasion")),
    ("NODES AND STAGE", ("regional_lymph_node_status", "pathologic_stage")),
    ("ANCILLARY STUDIES", ("molecular_classification",)),
)

_VALUE_TYPE_BY_SCHEMA = {
    "string": ValueType.STRING,
    "number": ValueType.NUMBER,
    "integer": ValueType.INTEGER,
    "boolean": ValueType.BOOLEAN,
}


def _shape_from_spec(spec: fields.FieldSpec) -> ValueShape:
    """The value shape of a checklist field, from its own declared schema (`fields.py`). Validating the
    stored value against the schema it was extracted against keeps a correctly-stored value from showing a
    spurious type or enum failure against a concept vocabulary it was never coded in."""
    schema = spec.value_schema
    value_type = _VALUE_TYPE_BY_SCHEMA.get(spec.value_type, ValueType.STRING)
    enum = frozenset(spec.enum_values) if spec.enum_values else None
    return ValueShape(
        value_type=value_type,
        enum=enum,
        minimum=schema.get("minimum"),
        maximum=schema.get("maximum"),
    )


def _primitive(value: object) -> object:
    """Coerce a checklist value to a JSON primitive: an enum to its value, everything else as-is."""
    return value.value if hasattr(value, "value") else value


def _extracted_from_field(concept_id: str, field: ChecklistField) -> ExtractedValue:
    """Turn one stored checklist field into the extraction record B1 validates. A not-applicable field is
    a coded absence (`not_applicable`); a flagged field was searched for and confirmed absent
    (`not_stated`); a field with a value is present. Silence is never the response to a miss."""
    if not field.applicable:
        state = FieldState.NOT_APPLICABLE
        value = None
    elif field.status is FieldStatus.FLAGGED:
        state = FieldState.NOT_STATED
        value = None
    else:
        state = FieldState.PRESENT
        value = _primitive(field.value)
    return ExtractedValue(concept_id=concept_id, value=value, state=state, evidence=field.evidence)


def _concept_stub(concept_id: str) -> SimpleNamespace:
    """The concept stand-in for a CAP field the canonical list does not model
    (`mapping.UNMODELED_CAP_FIELDS`, empty in the MVP). It carries the attributes `validate_field` reads,
    marking the value asserted and single-valued so its shape checks still run, while naming no
    derivation."""
    return SimpleNamespace(
        concept_id=concept_id,
        assertion_type=mapping.AssertionType.ASSERTED,
        cardinality=mapping.Cardinality.EXACTLY_ONE,
        derived=None,
    )


def validate_case_field(
    field_name: str,
    field: ChecklistField,
    *,
    compiled: dict[str, mapping.CompiledConcept],
    coverage: dict[str, str],
) -> tuple[Optional[str], FieldValidation]:
    """Validate one checklist field with the B1 pass (#97), returning its concept id (or None for an
    unmodeled field) and the validation block the confirmation card renders."""
    spec = fields.BY_NAME[field_name]
    shape = _shape_from_spec(spec)
    concept_id = coverage.get(field_name)
    if concept_id is not None and concept_id in compiled:
        concept: object = compiled[concept_id]
        extracted = _extracted_from_field(concept_id, field)
    else:
        concept = _concept_stub(field_name)
        extracted = _extracted_from_field(field_name, field)
    block = validate_field(extracted, concept, shape, {})  # type: ignore[arg-type]
    return concept_id, block


def _card_kind(field: ChecklistField, assertion_type: Optional[str]) -> str:
    """Which design-guideline section 9.3 card renders this field: the not-applicable card for a gated
    field, the derived-value card for a field whose concept is derived, the confirmed-value card
    otherwise."""
    if not field.applicable:
        return "not_applicable"
    if assertion_type == mapping.AssertionType.DERIVED.value:
        return "derived"
    return "confirmed"


def _attention(field: ChecklistField, block: FieldValidation, assertion_type: Optional[str]) -> str:
    """Whether a field reads as settled or needs the reviewer's judgment, directed by field type and
    resolution state rather than a confidence number (clinical-review-interface section 6; EndoExtract
    keeps fine-grained confidence off the screen).

    A confirmed field, and a not-applicable one, read `confident`. A derived field reads `needs_judgment`
    even when its shape is valid: a derivation is an interpretive call a reviewer should sanity-check. A
    field B1 could not promote reads `needs_judgment`, carrying the reason the block names."""
    if not field.applicable or field.status is FieldStatus.CONFIRMED:
        return "confident"
    if assertion_type == mapping.AssertionType.DERIVED.value:
        return "needs_judgment"
    if block.promotion is Promotion.CONFIRMED_ELIGIBLE:
        return "confident"
    return "needs_judgment"


def _enrich_derivation(derivation: dict, labels: dict[str, str]) -> dict:
    """Attach a human label to each derivation input concept id, so the derived card names its primitives
    rather than only their ids."""
    inputs = derivation.get("inputs") or []
    derivation["input_labels"] = [
        {"concept_id": cid, "label": labels.get(cid, cid)} for cid in inputs
    ]
    return derivation


def _field_block(
    field_name: str,
    field: ChecklistField,
    *,
    compiled: dict[str, mapping.CompiledConcept],
    coverage: dict[str, str],
    labels: dict[str, str],
    decision: Optional[dict],
) -> dict:
    spec = fields.BY_NAME[field_name]
    concept_id, block = validate_case_field(
        field_name, field, compiled=compiled, coverage=coverage
    )
    assertion_type = (
        compiled[concept_id].assertion_type.value
        if concept_id is not None and concept_id in compiled
        else None
    )
    block_json = block.model_dump(mode="json")
    block_json["derivation"] = _enrich_derivation(block_json["derivation"], labels)
    return {
        "field_name": field_name,
        "label": spec.label,
        "value": _primitive(field.value),
        "value_type": spec.value_type,
        "enum_values": spec.enum_values,
        "evidence": field.evidence.model_dump() if field.evidence else None,
        "status": field.status.value,
        "applicable": field.applicable,
        "not_applicable_reason": field.not_applicable_reason,
        "concept_id": concept_id,
        "concept_label": labels.get(concept_id) if concept_id else None,
        "assertion_type": assertion_type,
        "card_kind": _card_kind(field, assertion_type),
        "attention": _attention(field, block, assertion_type),
        "validation": block_json,
        "decision": decision,
    }


def _ledger_record(field_name: str, conf: dict, labels: dict[str, str], spec_label: str) -> dict:
    """One decision-ledger record (#98): the field decided, the answer filed, how it was reached, who
    filed it and when, and where it is filed, so a second reviewer reconstructs how the field reached its
    state."""
    options = json.loads(conf["options_json"]) if conf.get("options_json") else []
    value = json.loads(conf["value_json"]) if conf.get("value_json") is not None else None
    return {
        "field_name": field_name,
        "label": spec_label,
        "value": value,
        "status": conf["status"],
        "disposition": conf["disposition"],
        "decided_via": conf["decided_via"],
        "question": conf["question"],
        "options": options,
        "note": conf["note"],
        "confirmed_by": conf["confirmed_by"],
        "role": conf["role"],
        "confirmed_at": conf["confirmed_at"],
        # Where the answer is filed: the field on the case record, and the field_confirmations table that
        # carries the decision trail.
        "filed_to": {"field": field_name, "table": "field_confirmations"},
    }


def case_review(conn: Connection, case: Case, *, patient_filename: Optional[str], page_count: int) -> dict:
    """The case-review confirmation payload (#98): the case, its fields grouped into sections as record
    cards each carrying its B1 validation block, the confirmed count, and the decision ledger.

    Every value is served here, never hardcoded in the frontend; the frontend renders the record from this
    payload the way it renders the checklist from GET /api/fields."""
    compiled = {c.concept_id: c for c in mapping.compile_concepts()}
    coverage = mapping.cap_checklist_coverage()
    labels = crosswalk.concept_labels()
    decisions = storage.list_field_confirmations(conn, case.case_barcode)
    all_fields = case.checklist.all_fields()

    blocks: dict[str, dict] = {}
    for field_name, field in all_fields.items():
        blocks[field_name] = _field_block(
            field_name,
            field,
            compiled=compiled,
            coverage=coverage,
            labels=labels,
            decision=decisions.get(field_name),
        )

    placed: set[str] = set()
    sections_out: list[dict] = []
    for title, field_names in SECTIONS:
        section_fields = [blocks[name] for name in field_names if name in blocks]
        placed.update(name for name in field_names if name in blocks)
        if section_fields:
            sections_out.append({"title": title, "fields": section_fields})
    leftover = [blocks[name] for name in all_fields if name not in placed]
    if leftover:
        sections_out.append({"title": "OTHER", "fields": leftover})

    applicable = [b for b in blocks.values() if b["applicable"]]
    confirmed = [b for b in applicable if b["status"] == "confirmed"]
    eligible = [
        b for b in applicable
        if b["status"] != "confirmed" and b["validation"]["promotion"] == "confirmed_eligible"
    ]

    ledger = [
        _ledger_record(name, conf, labels, fields.BY_NAME[name].label)
        for name, conf in decisions.items()
        if name in fields.BY_NAME
    ]
    ledger.sort(key=lambda r: r["confirmed_at"], reverse=True)

    return {
        "case_barcode": case.case_barcode,
        "patient_filename": patient_filename,
        "report_date": case.report_date.isoformat() if case.report_date else None,
        "staging_edition": case.staging_edition.value,
        "status": case.status.value,
        "page_count": page_count,
        "sections": sections_out,
        "progress": {
            "confirmed": len(confirmed),
            "applicable": len(applicable),
            "confirmed_eligible": len(eligible),
            "total": len(blocks),
        },
        "ledger": ledger,
    }


# --- The value chat co-pilot (#98) ----------------------------------------------------------------
#
# Clicking a value opens a conversation scoped to it: the reviewer asks why the engine drafted the value
# it did, and the engine answers with its reasoning and the evidence it read. The exchange resolves into a
# filed decision rather than evaporating (the frontend files the engine's suggested value through the edit
# endpoint with decided_via='chat'). The engine call goes through a provider-agnostic seam, so the fast
# test suite runs against an injected fake with no API key. The chat model is named explicitly here rather
# than inherited from another pass.

CASE_CHAT_MODEL = "claude-sonnet-5"


@runtime_checkable
class FieldChatResponder(Protocol):
    """Answers a reviewer's question about one checklist field with the engine's reasoning and the
    evidence it read. The seam a local, on-premise model or a test fake plugs into: it never surfaces its
    provider to the call site."""

    def explain(self, *, field_context: dict, question: str, history: list[dict]) -> dict: ...


FIELD_CHAT_TOOL = {
    "name": "explain_field_value",
    "description": (
        "Answer the reviewer's question about one extracted checklist value: why the engine drafted it, "
        "what the evidence on the report supports, and, when the reviewer's question points at a "
        "correction, the value to file. Suggest only a value the field's schema permits, or null."
    ),
    "input_schema": {
        "type": "object",
        "properties": {
            "answer": {
                "type": "string",
                "description": "A direct answer to the reviewer's question, in plain declarative prose.",
            },
            "reasoning": {
                "type": "string",
                "description": "Why the value is drafted as it is: the reasoning from the report evidence "
                "and the field's definition to this value.",
            },
            "evidence": {
                "type": "array",
                "description": "The facts the answer rests on, each a fact and where it comes from.",
                "items": {
                    "type": "object",
                    "properties": {
                        "fact": {"type": "string"},
                        "source": {
                            "type": "string",
                            "description": "Where the fact comes from: the report evidence span, the "
                            "field definition, or the validation block.",
                        },
                    },
                    "required": ["fact", "source"],
                },
            },
            "suggested_value": {
                "description": "A value to re-file the field to, or null to leave it. A permitted value "
                "for a closed-vocabulary field; a number or string otherwise.",
            },
            "suggested_state": {
                "anyOf": [
                    {
                        "type": "string",
                        "enum": ["present", "not_stated", "indeterminate", "not_applicable"],
                    },
                    {"type": "null"},
                ],
                "description": "The coded state to file if the value is absent, or null to leave the field "
                "as drafted.",
            },
        },
        "required": ["answer", "reasoning", "evidence", "suggested_value", "suggested_state"],
    },
}

_CHAT_INSTRUCTIONS = """\
You are the extraction engine's co-pilot. A reviewer is confirming a value the engine drafted from a \
pathology report against the checklist, and asks you about one field. Answer their question about that \
field: why it holds the value it does, what the report evidence supports, and whether it should be \
corrected.

Ground every claim in the field context you are given: the drafted value, its evidence span from the \
report, the field definition and its permitted values, and the validation block. Do not invent evidence. \
When the reviewer's question points at a correction, set suggested_value to a value the field's schema \
permits; otherwise leave it null and explain why the drafted value stands. Call explain_field_value with \
your answer.\
"""


class AnthropicFieldChatResponder:
    """The hosted chat co-pilot: it reads the field context and answers the reviewer's question through a
    forced tool call. Names no provider at the call site."""

    def __init__(
        self,
        client: Optional[anthropic.Anthropic] = None,
        model: str = CASE_CHAT_MODEL,
        max_tokens: int = 1024,
    ) -> None:
        self._client = client
        self._model = model
        self._max_tokens = max_tokens

    def explain(self, *, field_context: dict, question: str, history: list[dict]) -> dict:
        client = self._client or get_client()
        system = [{"type": "text", "text": _CHAT_INSTRUCTIONS}]
        messages: list[dict] = []
        for turn in history:
            role = "assistant" if turn.get("role") == "assistant" else "user"
            messages.append({"role": role, "content": turn.get("text", "")})
        messages.append(
            {
                "role": "user",
                "content": (
                    "FIELD UNDER REVIEW:\n\n"
                    + json.dumps(field_context, indent=2)
                    + f"\n\nREVIEWER QUESTION:\n{question}\n\n"
                    "Call explain_field_value now."
                ),
            }
        )
        response = client.messages.create(
            model=self._model,
            max_tokens=self._max_tokens,
            system=system,
            tools=[FIELD_CHAT_TOOL],
            tool_choice={"type": "tool", "name": FIELD_CHAT_TOOL["name"]},
            messages=messages,
        )
        tool_call = next((b for b in response.content if b.type == "tool_use"), None)
        if tool_call is None:
            raise CaseReviewError("value chat returned no tool call")
        return tool_call.input


class StubFieldChatResponder:
    """A deterministic, offline chat responder for local demos and screenshot verification: it answers
    from the field context alone, with no model call and no API key. Enabled by the
    ENDOPATH_CASE_CHAT_STUB environment variable, never the default."""

    def explain(self, *, field_context: dict, question: str, history: list[dict]) -> dict:
        label = field_context.get("label", field_context.get("field_name"))
        value = field_context.get("value")
        evidence = field_context.get("evidence") or {}
        quote = evidence.get("quote") or ""
        if value is None:
            answer = (
                f"The engine drafted no value for '{label}'. The report evidence it read does not state "
                "it plainly, so the field is left for your judgment rather than guessed."
            )
        else:
            answer = (
                f"'{label}' reads {value!r} because the report states it"
                + (f': “{quote}”.' if quote else " in the evidence cited beside the value.")
            )
        derivation = field_context.get("validation", {}).get("derivation", {})
        reasoning_bits = [f"'{label}' is drafted from the evidence span cited on the page."]
        if derivation.get("status") not in (None, "not_derived"):
            reasoning_bits.append(
                f"Its concept is derived by the {derivation.get('rule')!r} rule; the derivation is "
                f"{derivation.get('status')}, so confirm it against the primitives on the page."
            )
        evidence_out = []
        if quote:
            evidence_out.append({"fact": quote, "source": "report evidence span"})
        evidence_out.append(
            {
                "fact": f"The field permits {field_context.get('enum_values') or 'an open vocabulary'}.",
                "source": "field definition",
            }
        )
        return {
            "answer": answer,
            "reasoning": " ".join(reasoning_bits),
            "evidence": evidence_out,
            "suggested_value": None,
            "suggested_state": None,
        }


class CaseReviewError(ValueError):
    """A value-chat request names a case or field the store does not carry, or a blank question."""


_CHAT_RESPONDER: Optional[FieldChatResponder] = None


def configure_field_chat_responder(responder: Optional[FieldChatResponder]) -> None:
    """Test/CLI hook to inject a chat responder (a fake in the fast suite), or reset to the default hosted
    one with None."""
    global _CHAT_RESPONDER
    _CHAT_RESPONDER = responder


def _field_chat_responder() -> FieldChatResponder:
    if _CHAT_RESPONDER is not None:
        return _CHAT_RESPONDER
    import os

    if os.environ.get("ENDOPATH_CASE_CHAT_STUB"):
        return StubFieldChatResponder()
    return AnthropicFieldChatResponder()


def _field_context(case: Case, field_name: str) -> dict:
    """The context the chat co-pilot reads: the field as drafted with its evidence, its definition and
    permitted values, and its validation block."""
    field = getattr(case.checklist, field_name)
    spec = fields.BY_NAME[field_name]
    compiled = {c.concept_id: c for c in mapping.compile_concepts()}
    coverage = mapping.cap_checklist_coverage()
    labels = crosswalk.concept_labels()
    concept_id, block = validate_case_field(field_name, field, compiled=compiled, coverage=coverage)
    return {
        "field_name": field_name,
        "label": spec.label,
        "value": _primitive(field.value),
        "value_type": spec.value_type,
        "enum_values": spec.enum_values,
        "evidence": field.evidence.model_dump() if field.evidence else None,
        "status": field.status.value,
        "applicable": field.applicable,
        "not_applicable_reason": field.not_applicable_reason,
        "concept_id": concept_id,
        "concept_label": labels.get(concept_id) if concept_id else None,
        "validation": block.model_dump(mode="json"),
    }


_CHAT_STATES = frozenset({"present", "not_stated", "indeterminate", "not_applicable"})


def explain_field(
    conn: Connection,
    *,
    case: Case,
    field_name: str,
    question: str,
    history: Optional[list[dict]] = None,
    responder: Optional[FieldChatResponder] = None,
) -> dict:
    """Answer a reviewer's question about one checklist field (#98): the engine's reasoning, the evidence
    it read, and, when the question points at a correction, the value to file. `conn` is unused today and
    kept for the endpoint's symmetry with the other case calls.

    Raises `CaseReviewError` on a blank question or a field the checklist does not carry. A suggested value
    outside the field's permitted set is dropped to null rather than raising: a chat suggestion is advice a
    reviewer files deliberately, not a write."""
    if not question.strip():
        raise CaseReviewError("a value chat message must carry a question")
    if field_name not in fields.BY_NAME:
        raise CaseReviewError(f"no checklist field named {field_name!r}")

    context = _field_context(case, field_name)
    responder = responder or _field_chat_responder()
    raw = responder.explain(field_context=context, question=question, history=history or [])

    answer = str(raw.get("answer") or "").strip()
    reasoning = str(raw.get("reasoning") or "").strip()
    evidence = [
        {"fact": str(item.get("fact", "")).strip(), "source": str(item.get("source", "")).strip()}
        for item in (raw.get("evidence") or [])
        if isinstance(item, dict) and item.get("fact")
    ]

    spec = fields.BY_NAME[field_name]
    suggested_value = raw.get("suggested_value")
    if suggested_value is not None and spec.enum_values and suggested_value not in spec.enum_values:
        suggested_value = None
    suggested_state = raw.get("suggested_state")
    if suggested_state is not None and suggested_state not in _CHAT_STATES:
        suggested_state = None

    return {
        "field_name": field_name,
        "answer": answer,
        "reasoning": reasoning,
        "evidence": evidence,
        "suggested_value": suggested_value,
        "suggested_state": suggested_state,
    }


def record_decision(
    conn: Connection,
    *,
    case_barcode: str,
    field: ChecklistField,
    field_name: str,
    confirmer: str,
    role: str,
    note: Optional[str],
    disposition: str,
    decided_via: Optional[str],
    question: Optional[str],
    options: Optional[list],
) -> None:
    """Write one field decision to the ledger (#98). The value recorded is the value the field now carries,
    the status is where the decision left it, and the ledger context (how the answer was reached, the
    question, the options) makes the row a source-to-decision record a second reviewer reconstructs."""
    storage.record_field_confirmation(
        conn,
        case_barcode=case_barcode,
        field_name=field_name,
        value_json=json.dumps(_primitive(field.value)),
        status=field.status.value,
        confirmed_by=confirmer,
        role=role,
        note=note,
        disposition=disposition,
        decided_via=decided_via,
        question=question,
        options_json=json.dumps(options) if options else None,
        confirmed_at=datetime.now(timezone.utc).isoformat(),
    )


__all__ = [
    "SECTIONS",
    "AnthropicFieldChatResponder",
    "CaseReviewError",
    "FieldChatResponder",
    "StubFieldChatResponder",
    "case_review",
    "configure_field_chat_responder",
    "explain_field",
    "record_decision",
    "validate_case_field",
]