"""Deterministic safety-layer eval: red-flag recall + benign specificity. Runs a fixed, committed vignette set through the rule engine and reports: - RECALL: each high-risk vignette (a lay paraphrase, not a verbatim rule pattern) must fire its expected rule. - SPECIFICITY: each benign vignette must fire zero rules. No model, no GPU, no network — judges can reproduce in seconds: python scripts/eval_safety.py The numbers quoted in README.md come from this script. If you edit data/red_flags.json, re-run this and update the README if the totals change. """ from __future__ import annotations import sys from dataclasses import dataclass, field from pathlib import Path from typing import Any sys.path.insert(0, str(Path(__file__).parent.parent)) from safety_rules import evaluate_red_flags # noqa: E402 from schema import StructuredIntake # noqa: E402 @dataclass(frozen=True) class Vignette: story: str expected_rule: str | None # None = benign, must fire NOTHING age: int | None = 40 meds: str = "" allergies: str = "" intake_kwargs: dict[str, Any] = field(default_factory=dict) VIGNETTES: list[Vignette] = [ # --- airway / deep-space infection (hard interrupt) --- Vignette("My neck is swollen and it is getting harder to breathe.", "airway_or_deep_space_infection"), Vignette("It is hard to swallow my spit and my jaw is clamped shut.", "airway_or_deep_space_infection"), Vignette("The floor of mouth is raising and I am choking on saliva.", "airway_or_deep_space_infection"), Vignette("i cant breathe and my face is blowing up", "airway_or_deep_space_infection"), Vignette("My throat feels like it is closing up since the tooth infection.", "airway_or_deep_space_infection"), # --- giant cell arteritis pattern (hard interrupt, age-gated) --- Vignette("My jaw cramps when chewing and my temple is tender.", "gca_jaw_claudication", age=60), Vignette( "Chewing makes the jaw pain start and resting makes it stop, plus my scalp hurts to touch.", "gca_jaw_claudication", age=62, ), Vignette("I have a new temple headache and my vision blurs sometimes.", "gca_jaw_claudication", age=58), Vignette("My doctor mentioned jaw claudication and told me to report it.", "gca_jaw_claudication", age=66), Vignette( "Jaw pain when chewing that eases with rest, and a new severe headache.", "gca_jaw_claudication", age=60, intake_kwargs={ "jaw_pain_with_chewing_relieved_by_rest": True, "vision_scalp_or_new_headache": True, }, ), # --- endodontic irrigant accident (hard interrupt) --- Vignette("My cheek swelled up right after root canal and it burns.", "hypochlorite_accident"), Vignette("Severe burning pain after root canal yesterday.", "hypochlorite_accident"), Vignette( "It looks bruised since the procedure.", "hypochlorite_accident", intake_kwargs={"bruising_or_burning_after_root_canal": True}, ), # --- loose crown / aspiration risk --- Vignette("My cap came off while eating.", "loose_crown_aspiration"), Vignette("The bridge is loose and about to fall off.", "loose_crown_aspiration"), Vignette("my crown is wobbly and i am scared i will swallow it", "loose_crown_aspiration"), # --- swelling / fever / abscess pattern --- Vignette("There is a gum boil with pus and I feel hot.", "facial_swelling_fever_abscess"), Vignette("My cheek is huge and I had chills last night.", "facial_swelling_fever_abscess"), Vignette("A pimple on my gum keeps draining with a bad taste.", "facial_swelling_fever_abscess"), # --- trauma with bite change --- Vignette("I fell on my jaw and now only one side touches.", "trauma_bite_change_fracture"), Vignette("After accident my teeth don't meet anymore.", "trauma_bite_change_fracture"), Vignette("got hit playing football and my bite suddenly changed", "trauma_bite_change_fracture"), # --- neurologic / cardiac mimic (hard interrupt) --- Vignette("My lip is numb since this morning and my speech is slurred.", "neurologic_or_cardiac_mimic"), Vignette("I get jaw pain when walking up stairs with chest pressure.", "neurologic_or_cardiac_mimic"), Vignette("Half my face is drooping since I woke up.", "neurologic_or_cardiac_mimic"), # --- severe uncontrolled pain --- Vignette("The pain is a 9 and nothing helps.", "severe_uncontrolled_pain"), Vignette("The toothache keeps me awake all night.", "severe_uncontrolled_pain"), Vignette( "It hurts badly tonight.", "severe_uncontrolled_pain", intake_kwargs={"pain_score": 9, "pain_prevents_sleep": True}, ), # --- post-extraction bleeding --- Vignette("Three hours after they pulled my tooth it is still bleeding.", "post_extraction_bleeding"), Vignette("Blood keeps gushing from the socket.", "post_extraction_bleeding"), Vignette("The bleeding wont stop even with gauze.", "post_extraction_bleeding"), # --- oroantral communication (mined from the builder's own case) --- Vignette( "Water comes out of my nose when I drink, ever since the extraction.", "oroantral_communication", ), Vignette("I feel air passing between my mouth and nose after the sinus repair.", "oroantral_communication"), Vignette("There is bubbling from the socket when I breathe.", "oroantral_communication"), # --- progressive tooth mobility (adult, age-gated) --- Vignette("My front tooth is getting looser by the week.", "progressive_tooth_mobility", age=45), Vignette("Two teeth feel loose and the gap is widening.", "progressive_tooth_mobility", age=51), Vignette("I have a wobbly tooth that was not wobbly last month.", "progressive_tooth_mobility", age=39), # --- repeated adjustments without relief (the builder's own #37 story) --- Vignette("They adjusted it five times and it still hits first.", "repeated_adjustments_without_relief"), Vignette("Multiple bite adjustments later, the bite still feels wrong.", "repeated_adjustments_without_relief"), Vignette("It still feels high after three separate visits.", "repeated_adjustments_without_relief"), # --- MRONJ medication prompt --- Vignette("Planning an extraction next month.", "mronj_medication_prompt", meds="Prolia injection"), Vignette("I take alendronate weekly for my bones.", "mronj_medication_prompt"), # --- medication-associated bruxism (compound: med AND jaw symptoms) --- Vignette( "I clench at night and wake with a tight jaw.", "medication_bruxism_link", meds="sertraline 50mg", ), Vignette( "Morning jaw stiffness, here for a check.", "medication_bruxism_link", meds="Vyvanse 30mg", intake_kwargs={"limited_opening_or_locked_jaw": True}, ), # --- avulsed permanent tooth (time-critical trauma) --- Vignette("My front tooth got knocked out when I fell off my bike.", "dental_trauma_avulsion"), Vignette("The whole tooth came out root and all after the hit.", "dental_trauma_avulsion"), # --- swallowed / aspirated tooth or dental fragment --- Vignette("My daughter swallowed her tooth and keeps coughing.", "swallowed_or_aspirated_object", age=6), Vignette("I swallowed a piece of my filling at dinner.", "swallowed_or_aspirated_object"), # --- immunocompromised context narrated in the story --- Vignette("I am on chemo and my gum is throbbing.", "immunocompromised_context"), # --- compound escalation: swelling AND fever together --- Vignette("My face is swollen and I am burning up.", "facial_swelling_with_fever"), # --- Egyptian Arabic dialect coverage (lay colloquial phrasings) --- Vignette("عندي ورم في رقبتي ومش عارف اتنفس", "airway_or_deep_space_infection", age=None), Vignette("حلقي بيقفل ومش عارف ابلع ريقي", "airway_or_deep_space_infection", age=None), Vignette("مش قادر اتنفس وخدي وارم", "airway_or_deep_space_infection", age=None), Vignette("بختنق وحاسس ان زوري بيقفل", "airway_or_deep_space_infection", age=None), Vignette("وقعت على وشي واتخلعت سنتي", "dental_trauma_avulsion", age=None), Vignette("ابني بلع سنة وهو بياكل", "swallowed_or_aspirated_object", age=6), Vignette("عندي وجع فكي مع المضغ وبيخف لما ارتاح", "gca_jaw_claudication", age=60), Vignette("بلعت الطربوش وانا باكل العشا", "loose_crown_aspiration", age=None), Vignette("وشي ورم بعد حشو العصب امبارح", "hypochlorite_accident", age=None), Vignette("خدي ورم فجأة وعندي سخونية", "facial_swelling_fever_abscess", age=None), # --- benign set: every one of these must fire ZERO rules --- Vignette("Routine cleaning appointment next week, no complaints.", None), Vignette("I want whiter teeth before my wedding.", None), Vignette("No swelling, no fever, just a small chip on a front tooth.", None), Vignette("My crown feels fine, I just want it checked.", None), Vignette("Mild cold sensitivity for a day after whitening.", None), Vignette("I don't have any pain and nothing feels wrong.", None), Vignette("My daughter lost a baby tooth naturally last week.", None, age=7), Vignette("My son's baby tooth fell out on its own, he is excited for the tooth fairy.", None, age=6), Vignette("I deny any trouble breathing, just cold sensitivity after whitening.", None), # Idiom / third-party traps — must NOT fire the trauma/ingestion/immuno rules. Vignette("I knocked out early from work to make this cleaning appointment.", None), Vignette("My dog swallowed it when my old retainer fell on the floor, so I need a new one.", None), Vignette("My aunt had chemo years ago; my own teeth just need a routine check.", None), Vignette("I pulled the weed out root and all this morning, then noticed mild cold sensitivity.", None), Vignette("Just booking a check-up before travelling. I take vitamins only.", None, meds="multivitamin"), # Benign Arabic: root-canal HISTORY must not fire the endodontic emergency rule. Vignette("عايز انضف سناني عند الدكتور", None, age=None), Vignette("عندي حشو عصب في ضرسي من سنتين ومفيش اي مشاكل", None, age=None), Vignette("عملت حشو عصب الاسبوع اللي فات وكله تمام", None, age=None), ] def run() -> int: recall_total = recall_hits = 0 benign_total = benign_clean = 0 failures: list[str] = [] for v in VIGNETTES: intake = StructuredIntake(chief_concern="eval", **v.intake_kwargs) findings = evaluate_red_flags(v.story, intake, age=v.age, meds=v.meds, allergies=v.allergies) fired = {f.rule_id for f in findings} if v.expected_rule is None: benign_total += 1 if not fired: benign_clean += 1 else: failures.append(f"FALSE POSITIVE {sorted(fired)} <- {v.story!r}") else: recall_total += 1 if v.expected_rule in fired: recall_hits += 1 else: failures.append(f"MISS {v.expected_rule} (fired={sorted(fired)}) <- {v.story!r}") print(f"Red-flag recall: {recall_hits}/{recall_total} lay-paraphrase vignettes") print(f"Benign specificity: {benign_clean}/{benign_total} benign stories with zero flags") for line in failures: print(" " + line) return 0 if not failures else 1 if __name__ == "__main__": sys.exit(run())