"""Unit tests for the deterministic grader.""" from email_env.grader import grade, grade_response_quality GOOD_RESPONSE = ( "We sincerely apologize for the duplicate charge on your account. " "Our billing team will investigate immediately and issue a refund " "within 3-5 business days. We will resolve this for you." ) EXPECTED = { "expected_category": "billing", "expected_priority": "high", "response_keywords": ["refund", "apologize", "billing", "resolve", "account"], } def test_perfect_action_scores_high(): score = grade( {"category": "billing", "priority": "high", "response": GOOD_RESPONSE}, EXPECTED, ) assert score >= 0.9 def test_all_wrong_scores_zero(): score = grade( {"category": "general", "priority": "low", "response": ""}, EXPECTED, ) assert score == 0.0 def test_empty_response_triggers_penalty(): score = grade( {"category": "billing", "priority": "high", "response": ""}, EXPECTED, ) # 0.4 + 0.3 - 0.2 penalty = 0.5 assert score == 0.5 def test_correct_category_only(): score = grade( {"category": "billing", "priority": "low", "response": GOOD_RESPONSE}, EXPECTED, ) assert 0.4 < score < 0.8 def test_correct_priority_only(): score = grade( {"category": "general", "priority": "high", "response": GOOD_RESPONSE}, EXPECTED, ) assert 0.3 < score < 0.7 def test_keyword_stuffing_is_penalised(): """Adversarial: just spam keywords. Should NOT score well.""" spam = "refund refund refund refund refund refund refund refund" score = grade_response_quality(spam, EXPECTED["response_keywords"]) assert score < 0.4, f"keyword stuffing scored {score}" def test_no_punctuation_is_penalised(): text = "we apologize and will refund your billing account resolve issue" score = grade_response_quality(text, EXPECTED["response_keywords"]) # has all keywords + structure but no punctuation -> coherence drops assert score < 0.95 def test_short_response_is_penalised(): text = "ok refund." score = grade_response_quality(text, EXPECTED["response_keywords"]) assert score < 0.5 def test_all_caps_is_penalised(): shout = "WE WILL APOLOGIZE AND REFUND YOUR BILLING ACCOUNT RESOLVE IMMEDIATELY." normal = "We will apologize and refund your billing account resolve immediately." shout_score = grade_response_quality(shout, EXPECTED["response_keywords"]) normal_score = grade_response_quality(normal, EXPECTED["response_keywords"]) assert shout_score < normal_score def test_good_response_without_all_keywords_still_scores(): text = ( "We are very sorry for the trouble. Our team will look into this " "and contact you shortly with a resolution." ) score = grade_response_quality(text, EXPECTED["response_keywords"]) # Has structure (ack + action) but few keywords assert score > 0.4 def test_missing_acknowledgement_lowers_score(): text = "Our team will refund your account and resolve the billing issue." score = grade_response_quality(text, EXPECTED["response_keywords"]) # Has action but no acknowledgement -> structural component halved perfect = grade_response_quality(GOOD_RESPONSE, EXPECTED["response_keywords"]) assert score < perfect def test_score_is_in_range(): for cat in ("billing", "technical", "general"): for pri in ("low", "medium", "high"): for resp in ("", "x", GOOD_RESPONSE, "refund " * 50): s = grade( {"category": cat, "priority": pri, "response": resp}, EXPECTED, ) assert 0.0 <= s <= 1.0, f"out of range: {s}" def test_grader_is_deterministic(): a = {"category": "billing", "priority": "high", "response": GOOD_RESPONSE} s1 = grade(a, EXPECTED) s2 = grade(a, EXPECTED) s3 = grade(a, EXPECTED) assert s1 == s2 == s3