File size: 3,754 Bytes
3be79c6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | import unittest
from app.ml.evaluate import evaluate_model
class FakeModel:
"""Scripted predictions per text, honoring the runtime contract."""
def __init__(self, answers):
self.answers = answers # text -> (label, score)
def predict(self, texts):
out = []
for t in texts:
if t not in self.answers:
return None # contract: None on failure
out.append(self.answers[t])
return out
class EvaluateModelTests(unittest.TestCase):
def setUp(self):
self.cases = [
{"text": "john.smith [SEP] acme.test", "expected": "NORMAL"},
{"text": "info [SEP] acme.test", "expected": "ROLE"},
{"text": "user [SEP] tempmail.example", "expected": "DISPOSABLE"},
{"text": "spamtrap [SEP] acme.test", "expected": "SPAM_TRAP"},
# A real-looking business domain the model must NOT flag.
{"text": "maria.garcia [SEP] harborview-law.test", "expected": "NORMAL"},
]
def test_per_class_metrics_are_returned(self):
model = FakeModel({
"john.smith [SEP] acme.test": ("NORMAL", 0.9),
"info [SEP] acme.test": ("ROLE", 0.9),
"user [SEP] tempmail.example": ("DISPOSABLE", 0.95),
"spamtrap [SEP] acme.test": ("SPAM_TRAP", 0.9),
"maria.garcia [SEP] harborview-law.test": ("NORMAL", 0.9),
})
r = evaluate_model(model, self.cases)
self.assertEqual(r["checked"], 5)
self.assertEqual(r["correct"], 5)
for cls in ("NORMAL", "DISPOSABLE", "ROLE", "SPAM_TRAP"):
self.assertIn(cls, r["per_class"])
self.assertEqual(r["per_class"][cls]["precision"], 1.0)
def test_false_disposable_destroys_the_gate(self):
# The single dangerous failure: flagging a legitimate business domain.
model = FakeModel({
"john.smith [SEP] acme.test": ("NORMAL", 0.9),
"info [SEP] acme.test": ("ROLE", 0.9),
"user [SEP] tempmail.example": ("DISPOSABLE", 0.95),
"spamtrap [SEP] acme.test": ("SPAM_TRAP", 0.9),
"maria.garcia [SEP] harborview-law.test": ("DISPOSABLE", 0.55),
})
r = evaluate_model(model, self.cases)
self.assertFalse(r["promotion_ready"])
self.assertEqual(r["wrong"], 1)
self.assertEqual(
r["dangerous_misses"],
["maria.garcia [SEP] harborview-law.test"],
"a NORMAL row predicted DISPOSABLE is the worst failure mode",
)
def test_promotion_requires_high_disposable_precision(self):
# 4/5 correct but the DISPOSABLE predictions are all right -> gate off
# only because sample support is tiny; precision alone decides.
model = FakeModel({
"john.smith [SEP] acme.test": ("NORMAL", 0.9),
"info [SEP] acme.test": ("ROLE", 0.9),
"user [SEP] tempmail.example": ("DISPOSABLE", 0.95),
"spamtrap [SEP] acme.test": ("SPAM_TRAP", 0.9),
"maria.garcia [SEP] harborview-law.test": ("NORMAL", 0.9),
})
r = evaluate_model(model, self.cases)
self.assertTrue(r["promotion_ready"])
self.assertEqual(r["per_class"]["DISPOSABLE"]["precision"], 1.0)
def test_model_failure_fails_the_gate(self):
model = FakeModel({}) # predict returns None for everything
r = evaluate_model(model, self.cases)
self.assertFalse(r["promotion_ready"])
self.assertEqual(r["checked"], 0)
def test_empty_cases_is_a_noop(self):
r = evaluate_model(FakeModel({}), [])
self.assertEqual(r["checked"], 0)
self.assertFalse(r["promotion_ready"])
if __name__ == "__main__":
unittest.main() |