File size: 2,947 Bytes
3be79c6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | """Evaluation harness and promotion gate for the email-risk model.
The single most dangerous failure mode is a **false DISPOSABLE**: flagging a
legitimate business domain as throwaway destroys a real lead and a credit. So
the promotion gate is deliberately stricter than "good accuracy":
promotion_ready = DISPOSABLE precision >= 0.98 AND zero dangerous misses
`dangerous_misses` lists every NORMAL case the model predicted DISPOSABLE -
those are reviewed individually before the model is ever allowed to block.
"""
from __future__ import annotations
from typing import Optional
PROMOTION_PRECISION = 0.98
LABELS = ("NORMAL", "DISPOSABLE", "ROLE", "SPAM_TRAP")
def evaluate_model(model, cases: list[dict]) -> dict:
"""Run `cases` ({"text", "expected"}) through the model and score it.
`model.predict(texts)` must honor the runtime contract:
list[tuple[label, confidence]] or None on failure.
"""
empty = {
"checked": 0, "correct": 0, "wrong": 0,
"per_class": {}, "dangerous_misses": [],
"promotion_ready": False,
"gate": f"DISPOSABLE precision >= {PROMOTION_PRECISION} and no dangerous misses",
}
if not cases:
return empty
texts = [c["text"] for c in cases]
try:
preds = model.predict(texts)
except Exception:
return empty
if preds is None or len(preds) != len(cases):
return empty
per_class: dict[str, dict[str, int]] = {
label: {"tp": 0, "fp": 0, "fn": 0, "support": 0} for label in LABELS
}
dangerous: list[str] = []
correct = 0
checked = 0
for case, (pred_label, _score) in zip(cases, preds):
expected = case.get("expected")
checked += 1
per_class.setdefault(expected, {"tp": 0, "fp": 0, "fn": 0, "support": 0})
per_class[expected]["support"] += 1
if pred_label == expected:
correct += 1
per_class[expected]["tp"] += 1
continue
per_class[pred_label]["fp"] += 1
per_class[expected]["fn"] += 1
if pred_label == "DISPOSABLE" and expected == "NORMAL":
dangerous.append(case["text"])
precision = {}
for label, m in per_class.items():
predicted = m["tp"] + m["fp"]
precision[label] = (m["tp"] / predicted) if predicted else 0.0
disposable_precision = precision.get("DISPOSABLE", 0.0)
promotion = disposable_precision >= PROMOTION_PRECISION and not dangerous
return {
"checked": checked,
"correct": correct,
"wrong": checked - correct,
"per_class": {
label: {
"precision": round(precision.get(label, 0.0), 4),
"tp": m["tp"], "fp": m["fp"], "fn": m["fn"], "support": m["support"],
}
for label, m in per_class.items()
},
"dangerous_misses": dangerous,
"promotion_ready": promotion,
"gate": empty["gate"],
} |