scripts/eval_safety.py

"""Deterministic safety-layer eval: red-flag recall + benign specificity.

Runs a fixed, committed vignette set through the rule engine and reports:
- RECALL: each high-risk vignette (a lay paraphrase, not a verbatim rule pattern)
  must fire its expected rule.
- SPECIFICITY: each benign vignette must fire zero rules.

No model, no GPU, no network — judges can reproduce in seconds:

    python scripts/eval_safety.py

The numbers quoted in README.md come from this script. If you edit
data/red_flags.json, re-run this and update the README if the totals change.
"""
from __future__ import annotations

import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

sys.path.insert(0, str(Path(__file__).parent.parent))

from safety_rules import evaluate_red_flags  # noqa: E402
from schema import StructuredIntake  # noqa: E402


@dataclass(frozen=True)
class Vignette:
    story: str
    expected_rule: str | None  # None = benign, must fire NOTHING
    age: int | None = 40
    meds: str = ""
    allergies: str = ""
    intake_kwargs: dict[str, Any] = field(default_factory=dict)


VIGNETTES: list[Vignette] = [
    # --- airway / deep-space infection (hard interrupt) ---
    Vignette("My neck is swollen and it is getting harder to breathe.", "airway_or_deep_space_infection"),
    Vignette("It is hard to swallow my spit and my jaw is clamped shut.", "airway_or_deep_space_infection"),
    Vignette("The floor of mouth is raising and I am choking on saliva.", "airway_or_deep_space_infection"),
    Vignette("i cant breathe and my face is blowing up", "airway_or_deep_space_infection"),
    Vignette("My throat feels like it is closing up since the tooth infection.", "airway_or_deep_space_infection"),
    # --- giant cell arteritis pattern (hard interrupt, age-gated) ---
    Vignette("My jaw cramps when chewing and my temple is tender.", "gca_jaw_claudication", age=60),
    Vignette(
        "Chewing makes the jaw pain start and resting makes it stop, plus my scalp hurts to touch.",
        "gca_jaw_claudication",
        age=62,
    ),
    Vignette("I have a new temple headache and my vision blurs sometimes.", "gca_jaw_claudication", age=58),
    Vignette("My doctor mentioned jaw claudication and told me to report it.", "gca_jaw_claudication", age=66),
    Vignette(
        "Jaw pain when chewing that eases with rest, and a new severe headache.",
        "gca_jaw_claudication",
        age=60,
        intake_kwargs={
            "jaw_pain_with_chewing_relieved_by_rest": True,
            "vision_scalp_or_new_headache": True,
        },
    ),
    # --- endodontic irrigant accident (hard interrupt) ---
    Vignette("My cheek swelled up right after root canal and it burns.", "hypochlorite_accident"),
    Vignette("Severe burning pain after root canal yesterday.", "hypochlorite_accident"),
    Vignette(
        "It looks bruised since the procedure.",
        "hypochlorite_accident",
        intake_kwargs={"bruising_or_burning_after_root_canal": True},
    ),
    # --- loose crown / aspiration risk ---
    Vignette("My cap came off while eating.", "loose_crown_aspiration"),
    Vignette("The bridge is loose and about to fall off.", "loose_crown_aspiration"),
    Vignette("my crown is wobbly and i am scared i will swallow it", "loose_crown_aspiration"),
    # --- swelling / fever / abscess pattern ---
    Vignette("There is a gum boil with pus and I feel hot.", "facial_swelling_fever_abscess"),
    Vignette("My cheek is huge and I had chills last night.", "facial_swelling_fever_abscess"),
    Vignette("A pimple on my gum keeps draining with a bad taste.", "facial_swelling_fever_abscess"),
    # --- trauma with bite change ---
    Vignette("I fell on my jaw and now only one side touches.", "trauma_bite_change_fracture"),
    Vignette("After accident my teeth don't meet anymore.", "trauma_bite_change_fracture"),
    Vignette("got hit playing football and my bite suddenly changed", "trauma_bite_change_fracture"),
    # --- neurologic / cardiac mimic (hard interrupt) ---
    Vignette("My lip is numb since this morning and my speech is slurred.", "neurologic_or_cardiac_mimic"),
    Vignette("I get jaw pain when walking up stairs with chest pressure.", "neurologic_or_cardiac_mimic"),
    Vignette("Half my face is drooping since I woke up.", "neurologic_or_cardiac_mimic"),
    # --- severe uncontrolled pain ---
    Vignette("The pain is a 9 and nothing helps.", "severe_uncontrolled_pain"),
    Vignette("The toothache keeps me awake all night.", "severe_uncontrolled_pain"),
    Vignette(
        "It hurts badly tonight.",
        "severe_uncontrolled_pain",
        intake_kwargs={"pain_score": 9, "pain_prevents_sleep": True},
    ),
    # --- post-extraction bleeding ---
    Vignette("Three hours after they pulled my tooth it is still bleeding.", "post_extraction_bleeding"),
    Vignette("Blood keeps gushing from the socket.", "post_extraction_bleeding"),
    Vignette("The bleeding wont stop even with gauze.", "post_extraction_bleeding"),
    # --- oroantral communication (mined from the builder's own case) ---
    Vignette(
        "Water comes out of my nose when I drink, ever since the extraction.",
        "oroantral_communication",
    ),
    Vignette("I feel air passing between my mouth and nose after the sinus repair.", "oroantral_communication"),
    Vignette("There is bubbling from the socket when I breathe.", "oroantral_communication"),
    # --- progressive tooth mobility (adult, age-gated) ---
    Vignette("My front tooth is getting looser by the week.", "progressive_tooth_mobility", age=45),
    Vignette("Two teeth feel loose and the gap is widening.", "progressive_tooth_mobility", age=51),
    Vignette("I have a wobbly tooth that was not wobbly last month.", "progressive_tooth_mobility", age=39),
    # --- repeated adjustments without relief (the builder's own #37 story) ---
    Vignette("They adjusted it five times and it still hits first.", "repeated_adjustments_without_relief"),
    Vignette("Multiple bite adjustments later, the bite still feels wrong.", "repeated_adjustments_without_relief"),
    Vignette("It still feels high after three separate visits.", "repeated_adjustments_without_relief"),
    # --- MRONJ medication prompt ---
    Vignette("Planning an extraction next month.", "mronj_medication_prompt", meds="Prolia injection"),
    Vignette("I take alendronate weekly for my bones.", "mronj_medication_prompt"),
    # --- medication-associated bruxism (compound: med AND jaw symptoms) ---
    Vignette(
        "I clench at night and wake with a tight jaw.",
        "medication_bruxism_link",
        meds="sertraline 50mg",
    ),
    Vignette(
        "Morning jaw stiffness, here for a check.",
        "medication_bruxism_link",
        meds="Vyvanse 30mg",
        intake_kwargs={"limited_opening_or_locked_jaw": True},
    ),
    # --- avulsed permanent tooth (time-critical trauma) ---
    Vignette("My front tooth got knocked out when I fell off my bike.", "dental_trauma_avulsion"),
    Vignette("The whole tooth came out root and all after the hit.", "dental_trauma_avulsion"),
    # --- swallowed / aspirated tooth or dental fragment ---
    Vignette("My daughter swallowed her tooth and keeps coughing.", "swallowed_or_aspirated_object", age=6),
    Vignette("I swallowed a piece of my filling at dinner.", "swallowed_or_aspirated_object"),
    # --- immunocompromised context narrated in the story ---
    Vignette("I am on chemo and my gum is throbbing.", "immunocompromised_context"),
    # --- compound escalation: swelling AND fever together ---
    Vignette("My face is swollen and I am burning up.", "facial_swelling_with_fever"),
    # --- Egyptian Arabic dialect coverage (lay colloquial phrasings) ---
    Vignette("عندي ورم في رقبتي ومش عارف اتنفس", "airway_or_deep_space_infection", age=None),
    Vignette("حلقي بيقفل ومش عارف ابلع ريقي", "airway_or_deep_space_infection", age=None),
    Vignette("مش قادر اتنفس وخدي وارم", "airway_or_deep_space_infection", age=None),
    Vignette("بختنق وحاسس ان زوري بيقفل", "airway_or_deep_space_infection", age=None),
    Vignette("وقعت على وشي واتخلعت سنتي", "dental_trauma_avulsion", age=None),
    Vignette("ابني بلع سنة وهو بياكل", "swallowed_or_aspirated_object", age=6),
    Vignette("عندي وجع فكي مع المضغ وبيخف لما ارتاح", "gca_jaw_claudication", age=60),
    Vignette("بلعت الطربوش وانا باكل العشا", "loose_crown_aspiration", age=None),
    Vignette("وشي ورم بعد حشو العصب امبارح", "hypochlorite_accident", age=None),
    Vignette("خدي ورم فجأة وعندي سخونية", "facial_swelling_fever_abscess", age=None),
    # --- benign set: every one of these must fire ZERO rules ---
    Vignette("Routine cleaning appointment next week, no complaints.", None),
    Vignette("I want whiter teeth before my wedding.", None),
    Vignette("No swelling, no fever, just a small chip on a front tooth.", None),
    Vignette("My crown feels fine, I just want it checked.", None),
    Vignette("Mild cold sensitivity for a day after whitening.", None),
    Vignette("I don't have any pain and nothing feels wrong.", None),
    Vignette("My daughter lost a baby tooth naturally last week.", None, age=7),
    Vignette("My son's baby tooth fell out on its own, he is excited for the tooth fairy.", None, age=6),
    Vignette("I deny any trouble breathing, just cold sensitivity after whitening.", None),
    # Idiom / third-party traps — must NOT fire the trauma/ingestion/immuno rules.
    Vignette("I knocked out early from work to make this cleaning appointment.", None),
    Vignette("My dog swallowed it when my old retainer fell on the floor, so I need a new one.", None),
    Vignette("My aunt had chemo years ago; my own teeth just need a routine check.", None),
    Vignette("I pulled the weed out root and all this morning, then noticed mild cold sensitivity.", None),
    Vignette("Just booking a check-up before travelling. I take vitamins only.", None, meds="multivitamin"),
    # Benign Arabic: root-canal HISTORY must not fire the endodontic emergency rule.
    Vignette("عايز انضف سناني عند الدكتور", None, age=None),
    Vignette("عندي حشو عصب في ضرسي من سنتين ومفيش اي مشاكل", None, age=None),
    Vignette("عملت حشو عصب الاسبوع اللي فات وكله تمام", None, age=None),
]


def run() -> int:
    recall_total = recall_hits = 0
    benign_total = benign_clean = 0
    failures: list[str] = []

    for v in VIGNETTES:
        intake = StructuredIntake(chief_concern="eval", **v.intake_kwargs)
        findings = evaluate_red_flags(v.story, intake, age=v.age, meds=v.meds, allergies=v.allergies)
        fired = {f.rule_id for f in findings}
        if v.expected_rule is None:
            benign_total += 1
            if not fired:
                benign_clean += 1
            else:
                failures.append(f"FALSE POSITIVE {sorted(fired)} <- {v.story!r}")
        else:
            recall_total += 1
            if v.expected_rule in fired:
                recall_hits += 1
            else:
                failures.append(f"MISS {v.expected_rule} (fired={sorted(fired)}) <- {v.story!r}")

    print(f"Red-flag recall:      {recall_hits}/{recall_total} lay-paraphrase vignettes")
    print(f"Benign specificity:   {benign_clean}/{benign_total} benign stories with zero flags")
    for line in failures:
        print("  " + line)
    return 0 if not failures else 1


if __name__ == "__main__":
    sys.exit(run())