tests/judge.py

import anthropic

_client = anthropic.Anthropic()
_MODEL = "claude-haiku-4-5-20251001"

_PROMPT = """\
Does the following response satisfy the criterion below? Reply with only PASS or FAIL.

Criterion: {criterion}

Response:
{response}"""


def judge(response: str, criterion: str) -> bool:
    """Use an LLM to evaluate whether a response satisfies a natural-language criterion."""
    result = _client.messages.create(
        model=_MODEL,
        max_tokens=16,
        messages=[{
            "role": "user",
            "content": _PROMPT.format(criterion=criterion, response=response),
        }],
    )
    return result.content[0].text.strip().upper().startswith("PASS")