import unittest from benchmarks.validators import ( validate_coding_output, validate_coding_retry, validate_mmlu_output, validate_reasoning_output, validate_reasoning_retry, ) class MMLUValidatorTests(unittest.TestCase): def test_accepts_explicit_answer_with_explanation(self): result = validate_mmlu_output( "ANSWER: C\nPerché la complessità nel caso peggiore è quadratica.", expected="C", ) self.assertTrue(result.valid) self.assertEqual(result.normalized, "C") self.assertTrue(result.evidence["correct"]) def test_accepts_marked_choice(self): result = validate_mmlu_output("La scelta corretta è (B). La stack segue LIFO.") self.assertTrue(result.valid) self.assertEqual(result.normalized, "B") def test_accepts_single_isolated_letter(self): result = validate_mmlu_output("D") self.assertTrue(result.valid) self.assertEqual(result.normalized, "D") def test_accepts_final_answer_contract_used_by_retry(self): result = validate_mmlu_output("Final answer: D\nThe two values overflow because both are negative.") self.assertTrue(result.valid) self.assertEqual(result.normalized, "D") def test_accepts_runner_bold_contract(self): result = validate_mmlu_output("**(B)** — risposta scelta") self.assertTrue(result.valid) self.assertEqual(result.normalized, "B") def test_explanation_letters_do_not_override_explicit_answer(self): result = validate_mmlu_output("ANSWER: A. Le opzioni B, C e D sono errate.") self.assertTrue(result.valid) self.assertEqual(result.normalized, "A") def test_rejects_missing_answer(self): result = validate_mmlu_output("La spiegazione descrive il concetto ma non seleziona un'opzione.") self.assertFalse(result.valid) self.assertEqual(result.failure_code, "answer_missing") def test_rejects_conflicting_explicit_answers(self): result = validate_mmlu_output("ANSWER: A\nFinal answer: C") self.assertFalse(result.valid) self.assertEqual(result.failure_code, "answer_ambiguous") self.assertEqual(result.evidence["distinct_candidates"], ["A", "C"]) def test_rejects_empty_output(self): result = validate_mmlu_output(None) self.assertFalse(result.valid) self.assertEqual(result.failure_code, "answer_missing") def test_expected_answer_only_affects_evidence(self): result = validate_mmlu_output("ANSWER: B", expected="C") self.assertTrue(result.valid) self.assertFalse(result.evidence["correct"]) self.assertEqual(result.normalized, "B") class CodingValidatorTests(unittest.TestCase): def test_accepts_typescript_fence_and_required_symbol(self): output = """Ecco l'implementazione: ```typescript export function reverseWords(value: string): string { return value.trim().split(/\\s+/).reverse().join(' '); } ``` """ result = validate_coding_output(output, required_symbols=["reverseWords"], min_significant_lines=3) self.assertTrue(result.valid) self.assertIn("reverseWords", result.normalized) self.assertEqual(result.evidence["missing_symbols"], []) def test_accepts_json_envelope(self): output = '{"language":"typescript","code":"export const add = (a: number, b: number): number => a + b;"}' result = validate_coding_output(output, required_symbols=["add"]) self.assertTrue(result.valid) self.assertEqual(result.evidence["extraction"], "json:code") def test_rejects_empty_fence(self): result = validate_coding_output("```typescript\n\n```") self.assertFalse(result.valid) self.assertEqual(result.failure_code, "code_empty") def test_rejects_missing_code_block(self): result = validate_coding_output("La soluzione è implementata nel testo seguente, ma il codice non è incluso.") self.assertFalse(result.valid) self.assertEqual(result.failure_code, "code_missing") def test_rejects_wrong_language_fence(self): result = validate_coding_output("```python\ndef add(a, b): return a + b\n```") self.assertFalse(result.valid) self.assertEqual(result.failure_code, "code_wrong_language") def test_rejects_required_symbol_missing(self): result = validate_coding_output( "```ts\nexport function subtract(a: number, b: number): number { return a - b; }\n```", required_symbols=["add"], ) self.assertFalse(result.valid) self.assertEqual(result.failure_code, "required_symbol_missing") self.assertEqual(result.evidence["missing_symbols"], ["add"]) def test_repaired_typescript_output_passes_contract(self): result = validate_coding_output( "```typescript\nexport function add(a: number, b: number): number {\n return a + b;\n}\n```", required_symbols=["add"], min_significant_lines=3, ) self.assertTrue(result.valid) self.assertIsNone(result.failure_code) def test_retry_is_requested_for_missing_typescript_before_last_attempt(self): result = validate_coding_retry( "code_correct: implementa TypeScript", "La spiegazione non contiene codice.", is_last_attempt=False, ) self.assertIsNotNone(result) self.assertEqual(result.failure_code, "code_missing") def test_retry_is_not_requested_on_last_attempt(self): result = validate_coding_retry( "code_correct: implementa TypeScript", "La spiegazione non contiene codice.", is_last_attempt=True, ) self.assertIsNone(result) def test_retry_is_not_requested_for_non_coding_goal(self): result = validate_coding_retry( "Scrivi una spiegazione concettuale", "La spiegazione non contiene codice.", is_last_attempt=False, ) self.assertIsNone(result) def test_retry_is_not_requested_for_valid_typescript(self): result = validate_coding_retry( "code_correct: implementa TypeScript", "```typescript\nexport const add = (a: number, b: number): number => a + b;\n```", is_last_attempt=False, ) self.assertIsNone(result) def test_rejects_placeholder_implementation(self): result = validate_coding_output( "```typescript\nexport function add(a: number, b: number): number {\n // TODO implement here\n return 0;\n}\n```", required_symbols=["add"], ) self.assertFalse(result.valid) self.assertEqual(result.failure_code, "code_placeholder") def test_rejects_non_typescript_prose_inside_fence(self): result = validate_coding_output("```typescript\nThis is only explanatory prose.\n```") self.assertFalse(result.valid) self.assertEqual(result.failure_code, "code_syntax_suspect") class ReasoningValidatorTests(unittest.TestCase): def test_accepts_gsm8k_contract_with_thousands_separator(self): result = validate_reasoning_output( "Somma i valori: 100 + 125 = 225.\n#### 225", expected=225, ) self.assertTrue(result.valid) self.assertEqual(result.normalized, "225") self.assertTrue(result.evidence["correct"]) def test_accepts_labeled_final_answer(self): result = validate_reasoning_output("I passaggi portano al totale. Final answer: 2,250", expected=2250) self.assertTrue(result.valid) self.assertEqual(result.normalized, "2250") def test_classifies_wrong_numeric_answer(self): result = validate_reasoning_output("Calcolo completo. #### 250", expected=225) self.assertFalse(result.valid) self.assertEqual(result.failure_code, "wrong_numeric_answer") self.assertFalse(result.evidence["correct"]) def test_classifies_missing_numeric_answer(self): result = validate_reasoning_output("La spiegazione termina senza un numero finale.", expected=225) self.assertFalse(result.valid) self.assertEqual(result.failure_code, "answer_missing") def test_classifies_conflicting_explicit_answers(self): result = validate_reasoning_output("#### 250\nFinal answer: 225", expected=225) self.assertFalse(result.valid) self.assertEqual(result.failure_code, "calculation_conflict") self.assertEqual(result.evidence["distinct_candidates"], [250, 225]) def test_reasoning_retry_is_requested_for_wrong_answer_before_last_attempt(self): result = validate_reasoning_retry( "reasoning GSM8K: risolvi il problema", "#### 250", expected=225, is_last_attempt=False, ) self.assertIsNotNone(result) self.assertEqual(result.failure_code, "wrong_numeric_answer") def test_reasoning_retry_is_not_requested_on_last_attempt(self): result = validate_reasoning_retry( "reasoning GSM8K: risolvi il problema", "#### 250", expected=225, is_last_attempt=True, ) self.assertIsNone(result) def test_reasoning_retry_is_not_requested_for_non_reasoning_goal(self): result = validate_reasoning_retry( "Implementa un componente TypeScript", "#### 250", expected=225, is_last_attempt=False, ) self.assertIsNone(result) if __name__ == "__main__": unittest.main()