Upload folder using huggingface_hub

.deploy-diag-weights.py

@@ -0,0 +1,36 @@
+"""Decisive: does the SAVED checkpoint reproduce the trainer's 0.999 eval?"""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+BACKEND = Path(r"C:\Users\Sitcd3\Documents\Rezerva DESIGN_Validator\Validator_Modul_X-test\email-validator\backend")
+
+import datasets  # noqa: E402
+import numpy as np  # noqa: E402
+from transformers import AutoModelForSequenceClassification, AutoTokenizer  # noqa: E402
+
+CKPT = str(BACKEND / "app" / "ml" / "runs" / "checkpoint-172")
+CSV = str(BACKEND / "app" / "ml" / "data" / "email_risk.csv")
+
+tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
+model = AutoModelForSequenceClassification.from_pretrained(CKPT)
+model.eval()
+
+raw = datasets.load_dataset("csv", data_files=CSV)["train"]
+split = raw.train_test_split(test_size=0.15, seed=42)
+test = split["test"]
+
+texts = [str(t) for t in test["text"]]
+enc = tokenizer(texts, padding="max_length", truncation=True,
+                max_length=64, return_tensors="pt")
+with __import__("torch").no_grad():
+    logits = model(**enc).logits
+pred = np.argmax(logits.numpy(), axis=-1)
+y = np.array(test["label"])
+
+acc = float((pred == y).mean())
+names = ["NORMAL", "DISPOSABLE", "ROLE", "SPAM_TRAP"]
+print(f"fresh-loaded checkpoint accuracy on held-out split: {acc:.4f}  (trainer said 0.999)")
+# Sample predictions vs truth:
+for i in range(8):
+    print(f"  {test['text'][i][:48]:50} true={names[y[i]]:10} pred={names[pred[i]]}")

.deploy-eval-onnx.py

@@ -0,0 +1,44 @@
+"""Smoke + generalization test of the exported ONNX model via the real runtime."""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from app.ml.runtime import EmailRiskModel  # noqa: E402
+
+MODEL_DIR = r"C:\Users\Sitcd3\Documents\Rezerva DESIGN_Validator\Validator_Modul_X-test\email-validator\backend\app\ml\onnx_model"
+
+CASES = [
+    # (text, expected, note)
+    ("user [SEP] mailinator.com", "DISPOSABLE", "blocklisted (seen in training)"),
+    ("someone [SEP] 10minutemail.com", "DISPOSABLE", "blocklisted (seen)"),
+    # UNSEEN disposable-style domains - never in any list. The whole point.
+    ("someone [SEP] temp-mail-pro.xyz", "DISPOSABLE", "UNSEEN disposable-style"),
+    ("someone [SEP] disposable-inbox-2024.top", "DISPOSABLE", "UNSEEN disposable-style"),
+    ("someone [SEP] tr-tempmail.net", "DISPOSABLE", "UNSEEN disposable-style"),
+    # Legitimate - must NOT be flagged.
+    ("john.smith [SEP] gmail.com", "NORMAL", "free provider"),
+    ("maria.garcia [SEP] harborview-law.test", "NORMAL", "business domain"),
+    ("kenji.tanaka [SEP] acme-corp.com", "NORMAL", "business domain"),
+    ("info [SEP] acme-corp.com", "ROLE", "role account"),
+]
+
+m = EmailRiskModel(repo=MODEL_DIR)
+ok = m.load()
+print("loaded:", ok)
+if not ok:
+    sys.exit(1)
+
+wrong = 0
+for text, expected, note in CASES:
+    pred = m.predict([text])
+    label = pred[0][0] if pred else "??"
+    score = pred[0][1] if pred else 0.0
+    mark = "OK " if label == expected else "WRONG"
+    if label != expected:
+        wrong += 1
+    print(f"{mark} {text:52} -> {label:10} {score:.3f}   ({note})")
+
+print()
+print(f"{len(CASES) - wrong}/{len(CASES)} correct")
+sys.exit(1 if wrong else 0)

.deploy-export-onnx.py

@@ -0,0 +1,32 @@
+"""Export the fine-tuned checkpoint to ONNX and bundle the tokenizer.
+
+Programmatic equivalent of `optimum-cli export onnx` (the CLI has a
+packaging issue on this Python build).
+"""
+import json
+import shutil
+import sys
+from pathlib import Path
+
+BACKEND = Path(__file__).resolve().parent
+CKPT = BACKEND / "app" / "ml" / "runs" / "checkpoint-172"
+OUT = BACKEND / "app" / "ml" / "onnx_model"
+
+from optimum.onnxruntime import ORTModelForSequenceClassification  # noqa: E402
+from transformers import AutoTokenizer  # noqa: E402
+
+print("loading checkpoint + exporting ONNX (CPU)...")
+model = ORTModelForSequenceClassification.from_pretrained(str(CKPT), export=True)
+tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
+
+OUT.mkdir(parents=True, exist_ok=True)
+model.save_pretrained(str(OUT))
+tokenizer.save_pretrained(str(OUT))
+
+files = sorted(f.name for f in OUT.iterdir() if f.is_file())
+total_mb = sum(f.stat().st_size for f in OUT.iterdir() if f.is_file()) / 1e6
+print(f"exported to {OUT}")
+for name in files:
+    print("  ", name)
+print(f"total: {total_mb:.1f} MB")
+print("config labels:", json.loads((OUT / "config.json").read_text(encoding="utf-8")).get("id2label"))

.deploy-test-pytorch.py

@@ -0,0 +1,32 @@
+"""Disambiguate: is the fine-tuned PyTorch model good, and only the ONNX
+export broken? Run the checkpoint directly through transformers."""
+import sys
+from pathlib import Path
+
+CKPT = Path(__file__).resolve().parent / "app" / "ml" / "runs" / "checkpoint-172"
+
+from transformers import AutoTokenizer, pipeline  # noqa: E402
+
+tok = AutoTokenizer.from_pretrained("distilbert-base-uncased")
+clf = pipeline("text-classification", model=str(CKPT), tokenizer=tok,
+               device="cpu", truncation=True, max_length=64)
+
+CASES = [
+    ("user [SEP] mailinator.com", "DISPOSABLE"),
+    ("someone [SEP] 10minutemail.com", "DISPOSABLE"),
+    ("someone [SEP] temp-mail-pro.xyz", "DISPOSABLE", ),
+    ("someone [SEP] disposable-inbox-2024.top", "DISPOSABLE"),
+    ("john.smith [SEP] gmail.com", "NORMAL"),
+    ("maria.garcia [SEP] harborview-law.test", "NORMAL"),
+    ("info [SEP] acme-corp.com", "ROLE"),
+]
+
+wrong = 0
+for text, expected in CASES:
+    r = clf(text)[0]
+    label, score = r["label"], r["score"]
+    ok = label == expected
+    wrong += 0 if ok else 1
+    print(f"{'OK ' if ok else 'WRONG'} {text:52} -> {label:10} {score:.3f}")
+print(f"\n{len(CASES)-wrong}/{len(CASES)} correct (PyTorch checkpoint)")
+sys.exit(1 if wrong else 0)

Dockerfile

@@ -10,6 +10,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     && rm -rf /var/lib/apt/lists/*
 
 # Install Python dependencies
+# torch CPU FIRST (the PyPI default wheel is the CUDA build, ~2.5 GB) — the
+# DistilBERT advisory layer runs on PyTorch (the ONNX export was unreliable).
+RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 

app/ml/evaluate.py

@@ -0,0 +1,89 @@
+"""Evaluation harness and promotion gate for the email-risk model.
+
+The single most dangerous failure mode is a **false DISPOSABLE**: flagging a
+legitimate business domain as throwaway destroys a real lead and a credit. So
+the promotion gate is deliberately stricter than "good accuracy":
+
+  promotion_ready = DISPOSABLE precision >= 0.98  AND  zero dangerous misses
+
+`dangerous_misses` lists every NORMAL case the model predicted DISPOSABLE -
+those are reviewed individually before the model is ever allowed to block.
+"""
+from __future__ import annotations
+
+from typing import Optional
+
+PROMOTION_PRECISION = 0.98
+LABELS = ("NORMAL", "DISPOSABLE", "ROLE", "SPAM_TRAP")
+
+
+def evaluate_model(model, cases: list[dict]) -> dict:
+    """Run `cases` ({"text", "expected"}) through the model and score it.
+
+    `model.predict(texts)` must honor the runtime contract:
+    list[tuple[label, confidence]] or None on failure.
+    """
+    empty = {
+        "checked": 0, "correct": 0, "wrong": 0,
+        "per_class": {}, "dangerous_misses": [],
+        "promotion_ready": False,
+        "gate": f"DISPOSABLE precision >= {PROMOTION_PRECISION} and no dangerous misses",
+    }
+    if not cases:
+        return empty
+
+    texts = [c["text"] for c in cases]
+    try:
+        preds = model.predict(texts)
+    except Exception:
+        return empty
+    if preds is None or len(preds) != len(cases):
+        return empty
+
+    per_class: dict[str, dict[str, int]] = {
+        label: {"tp": 0, "fp": 0, "fn": 0, "support": 0} for label in LABELS
+    }
+    dangerous: list[str] = []
+    correct = 0
+    checked = 0
+
+    for case, (pred_label, _score) in zip(cases, preds):
+        expected = case.get("expected")
+        checked += 1
+        per_class.setdefault(expected, {"tp": 0, "fp": 0, "fn": 0, "support": 0})
+        per_class[expected]["support"] += 1
+
+        if pred_label == expected:
+            correct += 1
+            per_class[expected]["tp"] += 1
+            continue
+
+        per_class[pred_label]["fp"] += 1
+        per_class[expected]["fn"] += 1
+
+        if pred_label == "DISPOSABLE" and expected == "NORMAL":
+            dangerous.append(case["text"])
+
+    precision = {}
+    for label, m in per_class.items():
+        predicted = m["tp"] + m["fp"]
+        precision[label] = (m["tp"] / predicted) if predicted else 0.0
+
+    disposable_precision = precision.get("DISPOSABLE", 0.0)
+    promotion = disposable_precision >= PROMOTION_PRECISION and not dangerous
+
+    return {
+        "checked": checked,
+        "correct": correct,
+        "wrong": checked - correct,
+        "per_class": {
+            label: {
+                "precision": round(precision.get(label, 0.0), 4),
+                "tp": m["tp"], "fp": m["fp"], "fn": m["fn"], "support": m["support"],
+            }
+            for label, m in per_class.items()
+        },
+        "dangerous_misses": dangerous,
+        "promotion_ready": promotion,
+        "gate": empty["gate"],
+    }

app/ml/model/config.json

@@ -0,0 +1,39 @@
+{
+  "activation": "gelu",
+  "architectures": [
+    "DistilBertForSequenceClassification"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": null,
+  "dim": 768,
+  "dropout": 0.1,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "hidden_dim": 3072,
+  "id2label": {
+    "0": "NORMAL",
+    "1": "DISPOSABLE",
+    "2": "ROLE",
+    "3": "SPAM_TRAP"
+  },
+  "initializer_range": 0.02,
+  "label2id": {
+    "DISPOSABLE": 1,
+    "NORMAL": 0,
+    "ROLE": 2,
+    "SPAM_TRAP": 3
+  },
+  "max_position_embeddings": 512,
+  "model_type": "distilbert",
+  "n_heads": 12,
+  "n_layers": 6,
+  "pad_token_id": 0,
+  "qa_dropout": 0.1,
+  "seq_classif_dropout": 0.2,
+  "sinusoidal_pos_embds": false,
+  "tie_weights_": true,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.19.0",
+  "use_cache": false,
+  "vocab_size": 30522
+}

app/ml/model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:212e338fe9e6d8cb89fce895dc21c306e6c61ab3d480922e82dd084e55283093
+size 267838720

app/ml/model/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

app/ml/model/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "DistilBertTokenizer",
+  "unk_token": "[UNK]"
+}

app/ml/runtime.py

@@ -14,22 +14,48 @@ from typing import Optional
 
 from ..config import settings
 
-# Optional at import time: the deployed image has onnxruntime, but dev/test
-# environments may not - and a missing runtime must read as "no ML", never as
-# a boot failure. Tests patch this module-level name.
-try:
-    from optimum.onnxruntime import ORTModelForSequenceClassification
-except Exception:  # pragma: no cover - runtime not installed
-    ORTModelForSequenceClassification = None
+def default_pipeline_factory(repo: str, token: str, batch_size: int):
+    """Production pipeline builder: transformers (PyTorch).
+
+    The ONNX export produced confidently-wrong labels under two transformers
+    versions while the PyTorch checkpoint scored 0.999 held-out - so inference
+    runs on the trained weights directly. transformers is imported lazily here:
+    a module-level import can fail under some import contexts (importlib_metadata
+    'Version' bug on Python 3.14), which must degrade to "no ML" only when
+    inference is actually attempted, not kill the capability at import time.
+    """
+    import transformers
+
+    tokenizer = transformers.AutoTokenizer.from_pretrained(
+        repo, token=token or None
+    )
+    model = transformers.AutoModelForSequenceClassification.from_pretrained(
+        repo, token=token or None
+    )
+    return transformers.pipeline(
+        "text-classification",
+        model=model,
+        tokenizer=tokenizer,
+        device="cpu",
+        truncation=True,
+        max_length=64,
+    )
 
 
 class EmailRiskModel:
     """Lazily-loaded ONNX text-classification wrapper. Fail-open by design."""
 
-    def __init__(self, repo: str, token: str = "", batch_size: int = 64):
+    def __init__(
+        self,
+        repo: str,
+        token: str = "",
+        batch_size: int = 64,
+        pipeline_factory: Optional[Callable[[str, str, int], object]] = None,
+    ):
         self.repo = repo
         self.token = token
         self.batch_size = batch_size
+        self._factory = pipeline_factory or default_pipeline_factory
         self._pipeline = None
         self._loaded = False
         self._lock = threading.Lock()
@@ -39,30 +65,13 @@ class EmailRiskModel:
         return self._loaded
 
     def load(self) -> bool:
-        """Download/load the ONNX model. Returns False on ANY failure."""
+        """Load the model. Returns False on ANY failure (fail-open design)."""
         with self._lock:
             if self._loaded:
                 return True
-            if ORTModelForSequenceClassification is None:
-                # onnxruntime/optimum not installed in this environment.
-                self._loaded = False
-                return False
             try:
-                from transformers import AutoTokenizer, pipeline  # noqa: PLC0415
-
-                tokenizer = AutoTokenizer.from_pretrained(
-                    self.repo, token=self.token or None
-                )
-                model = ORTModelForSequenceClassification.from_pretrained(
-                    self.repo, token=self.token or None
-                )
-                self._pipeline = pipeline(
-                    "text-classification",
-                    model=model,
-                    tokenizer=tokenizer,
-                    device="cpu",
-                    truncation=True,
-                    max_length=64,
+                self._pipeline = self._factory(
+                    self.repo, self.token, self.batch_size
                 )
                 self._loaded = True
                 return True

requirements.txt

@@ -38,3 +38,8 @@ python-jose[cryptography]>=3.3.0
 # --- Logging ---
 structlog>=24.4.0
 
+# --- DistilBERT advisory layer (PyTorch backend; torch installed separately
+#     from the CPU index in the Dockerfile) ---
+transformers>=4.41,<5
+safetensors>=0.4
+

tests/test_ml_evaluate.py

@@ -0,0 +1,92 @@
+import unittest
+
+from app.ml.evaluate import evaluate_model
+
+
+class FakeModel:
+    """Scripted predictions per text, honoring the runtime contract."""
+
+    def __init__(self, answers):
+        self.answers = answers  # text -> (label, score)
+
+    def predict(self, texts):
+        out = []
+        for t in texts:
+            if t not in self.answers:
+                return None  # contract: None on failure
+            out.append(self.answers[t])
+        return out
+
+
+class EvaluateModelTests(unittest.TestCase):
+    def setUp(self):
+        self.cases = [
+            {"text": "john.smith [SEP] acme.test", "expected": "NORMAL"},
+            {"text": "info [SEP] acme.test", "expected": "ROLE"},
+            {"text": "user [SEP] tempmail.example", "expected": "DISPOSABLE"},
+            {"text": "spamtrap [SEP] acme.test", "expected": "SPAM_TRAP"},
+            # A real-looking business domain the model must NOT flag.
+            {"text": "maria.garcia [SEP] harborview-law.test", "expected": "NORMAL"},
+        ]
+
+    def test_per_class_metrics_are_returned(self):
+        model = FakeModel({
+            "john.smith [SEP] acme.test": ("NORMAL", 0.9),
+            "info [SEP] acme.test": ("ROLE", 0.9),
+            "user [SEP] tempmail.example": ("DISPOSABLE", 0.95),
+            "spamtrap [SEP] acme.test": ("SPAM_TRAP", 0.9),
+            "maria.garcia [SEP] harborview-law.test": ("NORMAL", 0.9),
+        })
+        r = evaluate_model(model, self.cases)
+        self.assertEqual(r["checked"], 5)
+        self.assertEqual(r["correct"], 5)
+        for cls in ("NORMAL", "DISPOSABLE", "ROLE", "SPAM_TRAP"):
+            self.assertIn(cls, r["per_class"])
+            self.assertEqual(r["per_class"][cls]["precision"], 1.0)
+
+    def test_false_disposable_destroys_the_gate(self):
+        # The single dangerous failure: flagging a legitimate business domain.
+        model = FakeModel({
+            "john.smith [SEP] acme.test": ("NORMAL", 0.9),
+            "info [SEP] acme.test": ("ROLE", 0.9),
+            "user [SEP] tempmail.example": ("DISPOSABLE", 0.95),
+            "spamtrap [SEP] acme.test": ("SPAM_TRAP", 0.9),
+            "maria.garcia [SEP] harborview-law.test": ("DISPOSABLE", 0.55),
+        })
+        r = evaluate_model(model, self.cases)
+        self.assertFalse(r["promotion_ready"])
+        self.assertEqual(r["wrong"], 1)
+        self.assertEqual(
+            r["dangerous_misses"],
+            ["maria.garcia [SEP] harborview-law.test"],
+            "a NORMAL row predicted DISPOSABLE is the worst failure mode",
+        )
+
+    def test_promotion_requires_high_disposable_precision(self):
+        # 4/5 correct but the DISPOSABLE predictions are all right -> gate off
+        # only because sample support is tiny; precision alone decides.
+        model = FakeModel({
+            "john.smith [SEP] acme.test": ("NORMAL", 0.9),
+            "info [SEP] acme.test": ("ROLE", 0.9),
+            "user [SEP] tempmail.example": ("DISPOSABLE", 0.95),
+            "spamtrap [SEP] acme.test": ("SPAM_TRAP", 0.9),
+            "maria.garcia [SEP] harborview-law.test": ("NORMAL", 0.9),
+        })
+        r = evaluate_model(model, self.cases)
+        self.assertTrue(r["promotion_ready"])
+        self.assertEqual(r["per_class"]["DISPOSABLE"]["precision"], 1.0)
+
+    def test_model_failure_fails_the_gate(self):
+        model = FakeModel({})  # predict returns None for everything
+        r = evaluate_model(model, self.cases)
+        self.assertFalse(r["promotion_ready"])
+        self.assertEqual(r["checked"], 0)
+
+    def test_empty_cases_is_a_noop(self):
+        r = evaluate_model(FakeModel({}), [])
+        self.assertEqual(r["checked"], 0)
+        self.assertFalse(r["promotion_ready"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_ml_runtime.py

@@ -6,16 +6,47 @@ from app.ml.runtime import EmailRiskModel, get_model
 
 class RuntimeLoaderTests(unittest.TestCase):
     """The ML layer must be strictly optional: any failure degrades to "no ML"
-    and validation continues deterministically."""
+    and validation continues deterministically.
+
+    Backend = PyTorch transformers, NOT ONNX: the ONNX export produced
+    confidently-wrong labels under two transformers versions while the PyTorch
+    checkpoint scored 0.999 held-out (see VALIDATOR-PERFORMANCE-REPORT).
+    """
+
+    def test_load_uses_the_injected_factory(self):
+        # Dependency injection: the factory builds the pipeline; the runtime
+        # never imports transformers itself. This is what makes the ML layer
+        # testable without patching a lazy module boundary.
+        fake_pipe = MagicMock()
+        calls = {}
+
+        def factory(repo, token, batch_size):
+            calls["args"] = (repo, token, batch_size)
+            return fake_pipe
+
+        m = EmailRiskModel(repo="user/model", token="tok123",
+                           batch_size=32, pipeline_factory=factory)
+        self.assertTrue(m.load())
+        self.assertTrue(m.loaded)
+        self.assertEqual(calls["args"], ("user/model", "tok123", 32))
+        self.assertIs(m._pipeline, fake_pipe)
+
+    def test_factory_failure_returns_false_and_never_raises(self):
+        def bad_factory(repo, token, batch_size):
+            raise RuntimeError("hub down")
+
+        m = EmailRiskModel(repo="user/model", pipeline_factory=bad_factory)
+        self.assertFalse(m.load())
+        self.assertFalse(m.loaded)
+        self.assertIsNone(m._pipeline)
+
+    def test_load_uses_pytorch_transformers_by_default(self):
+        # The production factory loads from transformers (PyTorch - the ONNX
+        # export produced garbage predictions, so it was abandoned).
+        from app.ml.runtime import default_pipeline_factory
 
-    def test_load_failure_returns_false_and_never_raises(self):
         m = EmailRiskModel(repo="user/model", token="t")
-        with patch(
-            "app.ml.runtime.ORTModelForSequenceClassification",
-            side_effect=RuntimeError("hub down"),
-        ):
-            self.assertFalse(m.load())
-        self.assertFalse(m.loaded)
+        self.assertIs(m._factory, default_pipeline_factory)
 
     def test_predict_before_load_returns_none(self):
         m = EmailRiskModel(repo="user/model", token="t")