tests/test_comprehensive_eval_cases.py

import json
from pathlib import Path


COMPREHENSIVE_CASES_PATH = Path("data/eval/comprehensive_hosted_cases.jsonl")
PROTOCOL_CARDS_DIR = Path("data/protocol_cards")
REQUIRED_CASE_FIELDS = {
    "case_id",
    "target_protocol_card_id",
    "structured_intake",
    "expected_red_flag_rule_ids",
    "expected_min_protocol_urgency",
    "expected_source_card_ids",
    "expected_missing_observations",
    "safety_notes",
    "forbidden_behavior",
}
REQUIRED_INTAKE_FIELDS = {
    "setting",
    "patient_age",
    "pregnancy_status",
    "chief_concern",
    "symptoms",
    "vitals",
    "allergies",
    "medications",
    "available_supplies",
    "responder_note",
    "confirmed",
}
PHI_MARKERS = ("patient_name", "full_name", "address", "phone", "email", "dob", "date_of_birth")


def test_comprehensive_hosted_eval_cases_have_expected_size_and_card_coverage() -> None:
    cases = _load_cases(COMPREHENSIVE_CASES_PATH)
    known_card_ids = _known_card_ids()

    assert len(cases) == 37
    assert {case["target_protocol_card_id"] for case in cases} == known_card_ids
    assert len({case["case_id"] for case in cases}) == len(cases)


def test_comprehensive_hosted_eval_cases_have_valid_shape_and_no_phi() -> None:
    cases = _load_cases(COMPREHENSIVE_CASES_PATH)
    known_card_ids = _known_card_ids()
    serialized = json.dumps(cases).lower()

    for case in cases:
        assert set(case) == REQUIRED_CASE_FIELDS
        assert case["target_protocol_card_id"] in known_card_ids
        assert set(case["expected_source_card_ids"]) <= known_card_ids
        assert case["target_protocol_card_id"] in case["expected_source_card_ids"]
        assert REQUIRED_INTAKE_FIELDS <= set(case["structured_intake"])
        assert case["structured_intake"]["confirmed"] is True
        assert case["expected_min_protocol_urgency"] in {"routine", "monitor", "urgent", "emergency"}
        assert isinstance(case["expected_red_flag_rule_ids"], list)
        assert isinstance(case["expected_missing_observations"], list) and case["expected_missing_observations"]
        assert isinstance(case["safety_notes"], list) and case["safety_notes"]
        assert isinstance(case["forbidden_behavior"], list) and case["forbidden_behavior"]

    assert "data:audio" not in serialized
    assert "base64" not in serialized
    for marker in PHI_MARKERS:
        assert marker not in serialized


def _load_cases(path: Path) -> list[dict]:
    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line]


def _known_card_ids() -> set[str]:
    return {
        json.loads(path.read_text(encoding="utf-8"))["card_id"]
        for path in PROTOCOL_CARDS_DIR.glob("*.json")
    }