src/endopath/validation_llm.py

"""The hosted-model adjunct to the B1 validation pass (#97).

The deterministic checks in `validation.py` (type, enum, cardinality, derivation recompute) run
with no model call. This module adds the one judgment the deterministic pass cannot make on its own:
whether the extracted evidence span actually supports the extracted value. It asks the validation
model to adjudicate one field through a forced tool call, the same schema-constrained pattern
`llm_extraction.py` uses, so the model returns a structured verdict rather than free text.

The model is configured explicitly here (`VALIDATION_MODEL`), not inherited from the extraction
model: the extraction and the validation of that extraction are deliberately separate calls. The
fast test suite mocks the client, so no API key is needed; the live call is exercised behind a
`@pytest.mark.slow` test.
"""

from __future__ import annotations

from typing import Optional

import anthropic

from endopath.llm_extraction import get_client

# The validation model, set explicitly (claude-api skill: default to Claude Opus 4.8). Kept
# distinct from llm_extraction.MODEL so the validation pass and the extraction it validates are
# separately configurable.
VALIDATION_MODEL = "claude-sonnet-5"

# The forced tool the model must call: a structured verdict on one field, never free text.
FIELD_REVIEW_TOOL = {
    "name": "record_field_review",
    "description": (
        "Record whether the extracted evidence span supports the extracted value for one "
        "checklist concept."
    ),
    "input_schema": {
        "type": "object",
        "properties": {
            "agrees": {
                "type": "boolean",
                "description": "True if the evidence span supports the extracted value.",
            },
            "note": {
                "type": "string",
                "description": "A one-sentence reason, citing the evidence where it disagrees.",
            },
        },
        "required": ["agrees", "note"],
    },
}

_PROMPT_TEMPLATE = """\
You are validating one field of a structured extraction from a pathology report against the \
evidence that justified it. The deterministic checks (type, enum, cardinality, and the derivation \
recompute) have already run; your job is the judgment they cannot make: does the cited evidence \
actually support this value?

Call record_field_review with your verdict.

concept: {concept_id}
extracted value: {value!r}
evidence span: {evidence!r}
"""


def review_field(
    client: Optional[anthropic.Anthropic],
    *,
    concept_id: str,
    value: object,
    evidence: Optional[str] = None,
    model: str = VALIDATION_MODEL,
) -> dict:
    """Ask the validation model whether the evidence supports the value. Returns the tool input
    (`{"agrees": bool, "note": str}`). Raises on a truncated tool call, the same failure
    `llm_extraction.extract_raw` guards against."""
    client = client or get_client()
    response = client.messages.create(
        model=model,
        max_tokens=1024,
        tools=[FIELD_REVIEW_TOOL],
        tool_choice={"type": "tool", "name": "record_field_review"},
        messages=[
            {
                "role": "user",
                "content": _PROMPT_TEMPLATE.format(
                    concept_id=concept_id, value=value, evidence=evidence
                ),
            }
        ],
    )
    if response.stop_reason == "max_tokens":
        raise RuntimeError(
            "field review hit max_tokens before completing the tool call; raise max_tokens."
        )
    tool_call = next(block for block in response.content if block.type == "tool_use")
    return tool_call.input