evaluation/judge_request.py

"""Construct one DDI judge request and validate its response; no network calls."""
import json
from inference.evaluation.protocol import REPO_ROOT, prompt_text


def build_judge_request(reference_diagnosis, model_output):
    if not isinstance(reference_diagnosis, str) or not reference_diagnosis.strip():
        raise ValueError('A nonempty reference diagnosis is required for scoring.')
    if not isinstance(model_output, str):
        raise ValueError('The generated response must be a string.')
    config = json.loads((REPO_ROOT / 'configs/ddi_judge.json').read_text())
    tail = model_output[-config['batch_evaluator_defaults']['model_output_tail_characters']:]
    user = prompt_text('ddi_judge_user.txt').format(
        ground_truth=reference_diagnosis, model_output=tail
    )
    return {
        key: config[key] for key in ['model', 'temperature', 'max_tokens', 'response_format', 'extra_body']
    } | {'messages': [
        {'role': 'system', 'content': prompt_text('ddi_judge_system.txt')},
        {'role': 'user', 'content': user},
    ]}


def validate_judge_response(content):
    result = json.loads(content)
    if not isinstance(result, dict) or set(result) != {'predicted_diagnosis', 'correct', 'reason'}:
        raise ValueError('Unexpected judge response fields.')
    if not isinstance(result['predicted_diagnosis'], str) or type(result['correct']) is not bool:
        raise ValueError('Invalid diagnosis or correctness type.')
    if not isinstance(result['reason'], str) or not result['reason'].strip():
        raise ValueError('A nonempty reason is required.')
    if result['correct'] and not result['predicted_diagnosis'].strip():
        raise ValueError('A correct response requires an identified final diagnosis.')
    return result