| """Evaluation harness and promotion gate for the email-risk model. |
| |
| The single most dangerous failure mode is a **false DISPOSABLE**: flagging a |
| legitimate business domain as throwaway destroys a real lead and a credit. So |
| the promotion gate is deliberately stricter than "good accuracy": |
| |
| promotion_ready = DISPOSABLE precision >= 0.98 AND zero dangerous misses |
| |
| `dangerous_misses` lists every NORMAL case the model predicted DISPOSABLE - |
| those are reviewed individually before the model is ever allowed to block. |
| """ |
| from __future__ import annotations |
|
|
| from typing import Optional |
|
|
| PROMOTION_PRECISION = 0.98 |
| LABELS = ("NORMAL", "DISPOSABLE", "ROLE", "SPAM_TRAP") |
|
|
|
|
| def evaluate_model(model, cases: list[dict]) -> dict: |
| """Run `cases` ({"text", "expected"}) through the model and score it. |
| |
| `model.predict(texts)` must honor the runtime contract: |
| list[tuple[label, confidence]] or None on failure. |
| """ |
| empty = { |
| "checked": 0, "correct": 0, "wrong": 0, |
| "per_class": {}, "dangerous_misses": [], |
| "promotion_ready": False, |
| "gate": f"DISPOSABLE precision >= {PROMOTION_PRECISION} and no dangerous misses", |
| } |
| if not cases: |
| return empty |
|
|
| texts = [c["text"] for c in cases] |
| try: |
| preds = model.predict(texts) |
| except Exception: |
| return empty |
| if preds is None or len(preds) != len(cases): |
| return empty |
|
|
| per_class: dict[str, dict[str, int]] = { |
| label: {"tp": 0, "fp": 0, "fn": 0, "support": 0} for label in LABELS |
| } |
| dangerous: list[str] = [] |
| correct = 0 |
| checked = 0 |
|
|
| for case, (pred_label, _score) in zip(cases, preds): |
| expected = case.get("expected") |
| checked += 1 |
| per_class.setdefault(expected, {"tp": 0, "fp": 0, "fn": 0, "support": 0}) |
| per_class[expected]["support"] += 1 |
|
|
| if pred_label == expected: |
| correct += 1 |
| per_class[expected]["tp"] += 1 |
| continue |
|
|
| per_class[pred_label]["fp"] += 1 |
| per_class[expected]["fn"] += 1 |
|
|
| if pred_label == "DISPOSABLE" and expected == "NORMAL": |
| dangerous.append(case["text"]) |
|
|
| precision = {} |
| for label, m in per_class.items(): |
| predicted = m["tp"] + m["fp"] |
| precision[label] = (m["tp"] / predicted) if predicted else 0.0 |
|
|
| disposable_precision = precision.get("DISPOSABLE", 0.0) |
| promotion = disposable_precision >= PROMOTION_PRECISION and not dangerous |
|
|
| return { |
| "checked": checked, |
| "correct": correct, |
| "wrong": checked - correct, |
| "per_class": { |
| label: { |
| "precision": round(precision.get(label, 0.0), 4), |
| "tp": m["tp"], "fp": m["fp"], "fn": m["fn"], "support": m["support"], |
| } |
| for label, m in per_class.items() |
| }, |
| "dangerous_misses": dangerous, |
| "promotion_ready": promotion, |
| "gate": empty["gate"], |
| } |
