Spaces:

sitcod3
/

modul-x

Running

modul-x / tests /test_ml_evaluate.py

Upload folder using huggingface_hub

3be79c6 verified 9 days ago

3.75 kB

import unittest

from app.ml.evaluate import evaluate_model


class FakeModel:
    """Scripted predictions per text, honoring the runtime contract."""

    def __init__(self, answers):
        self.answers = answers  # text -> (label, score)

    def predict(self, texts):
        out = []
        for t in texts:
            if t not in self.answers:
                return None  # contract: None on failure
            out.append(self.answers[t])
        return out


class EvaluateModelTests(unittest.TestCase):
    def setUp(self):
        self.cases = [
            {"text": "john.smith [SEP] acme.test", "expected": "NORMAL"},
            {"text": "info [SEP] acme.test", "expected": "ROLE"},
            {"text": "user [SEP] tempmail.example", "expected": "DISPOSABLE"},
            {"text": "spamtrap [SEP] acme.test", "expected": "SPAM_TRAP"},
            # A real-looking business domain the model must NOT flag.
            {"text": "maria.garcia [SEP] harborview-law.test", "expected": "NORMAL"},
        ]

    def test_per_class_metrics_are_returned(self):
        model = FakeModel({
            "john.smith [SEP] acme.test": ("NORMAL", 0.9),
            "info [SEP] acme.test": ("ROLE", 0.9),
            "user [SEP] tempmail.example": ("DISPOSABLE", 0.95),
            "spamtrap [SEP] acme.test": ("SPAM_TRAP", 0.9),
            "maria.garcia [SEP] harborview-law.test": ("NORMAL", 0.9),
        })
        r = evaluate_model(model, self.cases)
        self.assertEqual(r["checked"], 5)
        self.assertEqual(r["correct"], 5)
        for cls in ("NORMAL", "DISPOSABLE", "ROLE", "SPAM_TRAP"):
            self.assertIn(cls, r["per_class"])
            self.assertEqual(r["per_class"][cls]["precision"], 1.0)

    def test_false_disposable_destroys_the_gate(self):
        # The single dangerous failure: flagging a legitimate business domain.
        model = FakeModel({
            "john.smith [SEP] acme.test": ("NORMAL", 0.9),
            "info [SEP] acme.test": ("ROLE", 0.9),
            "user [SEP] tempmail.example": ("DISPOSABLE", 0.95),
            "spamtrap [SEP] acme.test": ("SPAM_TRAP", 0.9),
            "maria.garcia [SEP] harborview-law.test": ("DISPOSABLE", 0.55),
        })
        r = evaluate_model(model, self.cases)
        self.assertFalse(r["promotion_ready"])
        self.assertEqual(r["wrong"], 1)
        self.assertEqual(
            r["dangerous_misses"],
            ["maria.garcia [SEP] harborview-law.test"],
            "a NORMAL row predicted DISPOSABLE is the worst failure mode",
        )

    def test_promotion_requires_high_disposable_precision(self):
        # 4/5 correct but the DISPOSABLE predictions are all right -> gate off
        # only because sample support is tiny; precision alone decides.
        model = FakeModel({
            "john.smith [SEP] acme.test": ("NORMAL", 0.9),
            "info [SEP] acme.test": ("ROLE", 0.9),
            "user [SEP] tempmail.example": ("DISPOSABLE", 0.95),
            "spamtrap [SEP] acme.test": ("SPAM_TRAP", 0.9),
            "maria.garcia [SEP] harborview-law.test": ("NORMAL", 0.9),
        })
        r = evaluate_model(model, self.cases)
        self.assertTrue(r["promotion_ready"])
        self.assertEqual(r["per_class"]["DISPOSABLE"]["precision"], 1.0)

    def test_model_failure_fails_the_gate(self):
        model = FakeModel({})  # predict returns None for everything
        r = evaluate_model(model, self.cases)
        self.assertFalse(r["promotion_ready"])
        self.assertEqual(r["checked"], 0)

    def test_empty_cases_is_a_noop(self):
        r = evaluate_model(FakeModel({}), [])
        self.assertEqual(r["checked"], 0)
        self.assertFalse(r["promotion_ready"])


if __name__ == "__main__":
    unittest.main()

	import unittest

	from app.ml.evaluate import evaluate_model


	class FakeModel:
	"""Scripted predictions per text, honoring the runtime contract."""

	def __init__(self, answers):
	self.answers = answers # text -> (label, score)

	def predict(self, texts):
	out = []
	for t in texts:
	if t not in self.answers:
	return None # contract: None on failure
	out.append(self.answers[t])
	return out


	class EvaluateModelTests(unittest.TestCase):
	def setUp(self):
	self.cases = [
	{"text": "john.smith [SEP] acme.test", "expected": "NORMAL"},
	{"text": "info [SEP] acme.test", "expected": "ROLE"},
	{"text": "user [SEP] tempmail.example", "expected": "DISPOSABLE"},
	{"text": "spamtrap [SEP] acme.test", "expected": "SPAM_TRAP"},
	# A real-looking business domain the model must NOT flag.
	{"text": "maria.garcia [SEP] harborview-law.test", "expected": "NORMAL"},
	]

	def test_per_class_metrics_are_returned(self):
	model = FakeModel({
	"john.smith [SEP] acme.test": ("NORMAL", 0.9),
	"info [SEP] acme.test": ("ROLE", 0.9),
	"user [SEP] tempmail.example": ("DISPOSABLE", 0.95),
	"spamtrap [SEP] acme.test": ("SPAM_TRAP", 0.9),
	"maria.garcia [SEP] harborview-law.test": ("NORMAL", 0.9),
	})
	r = evaluate_model(model, self.cases)
	self.assertEqual(r["checked"], 5)
	self.assertEqual(r["correct"], 5)
	for cls in ("NORMAL", "DISPOSABLE", "ROLE", "SPAM_TRAP"):
	self.assertIn(cls, r["per_class"])
	self.assertEqual(r["per_class"][cls]["precision"], 1.0)

	def test_false_disposable_destroys_the_gate(self):
	# The single dangerous failure: flagging a legitimate business domain.
	model = FakeModel({
	"john.smith [SEP] acme.test": ("NORMAL", 0.9),
	"info [SEP] acme.test": ("ROLE", 0.9),
	"user [SEP] tempmail.example": ("DISPOSABLE", 0.95),
	"spamtrap [SEP] acme.test": ("SPAM_TRAP", 0.9),
	"maria.garcia [SEP] harborview-law.test": ("DISPOSABLE", 0.55),
	})
	r = evaluate_model(model, self.cases)
	self.assertFalse(r["promotion_ready"])
	self.assertEqual(r["wrong"], 1)
	self.assertEqual(
	r["dangerous_misses"],
	["maria.garcia [SEP] harborview-law.test"],
	"a NORMAL row predicted DISPOSABLE is the worst failure mode",
	)

	def test_promotion_requires_high_disposable_precision(self):
	# 4/5 correct but the DISPOSABLE predictions are all right -> gate off
	# only because sample support is tiny; precision alone decides.
	model = FakeModel({
	"john.smith [SEP] acme.test": ("NORMAL", 0.9),
	"info [SEP] acme.test": ("ROLE", 0.9),
	"user [SEP] tempmail.example": ("DISPOSABLE", 0.95),
	"spamtrap [SEP] acme.test": ("SPAM_TRAP", 0.9),
	"maria.garcia [SEP] harborview-law.test": ("NORMAL", 0.9),
	})
	r = evaluate_model(model, self.cases)
	self.assertTrue(r["promotion_ready"])
	self.assertEqual(r["per_class"]["DISPOSABLE"]["precision"], 1.0)

	def test_model_failure_fails_the_gate(self):
	model = FakeModel({}) # predict returns None for everything
	r = evaluate_model(model, self.cases)
	self.assertFalse(r["promotion_ready"])
	self.assertEqual(r["checked"], 0)

	def test_empty_cases_is_a_noop(self):
	r = evaluate_model(FakeModel({}), [])
	self.assertEqual(r["checked"], 0)
	self.assertFalse(r["promotion_ready"])


	if __name__ == "__main__":
	unittest.main()