tests/test_benchmark_validators.py

import unittest

from benchmarks.validators import (
    validate_coding_output,
    validate_coding_retry,
    validate_mmlu_output,
    validate_reasoning_output,
    validate_reasoning_retry,
)


class MMLUValidatorTests(unittest.TestCase):
    def test_accepts_explicit_answer_with_explanation(self):
        result = validate_mmlu_output(
            "ANSWER: C\nPerché la complessità nel caso peggiore è quadratica.",
            expected="C",
        )
        self.assertTrue(result.valid)
        self.assertEqual(result.normalized, "C")
        self.assertTrue(result.evidence["correct"])

    def test_accepts_marked_choice(self):
        result = validate_mmlu_output("La scelta corretta è (B). La stack segue LIFO.")
        self.assertTrue(result.valid)
        self.assertEqual(result.normalized, "B")

    def test_accepts_single_isolated_letter(self):
        result = validate_mmlu_output("D")
        self.assertTrue(result.valid)
        self.assertEqual(result.normalized, "D")

    def test_accepts_final_answer_contract_used_by_retry(self):
        result = validate_mmlu_output("Final answer: D\nThe two values overflow because both are negative.")
        self.assertTrue(result.valid)
        self.assertEqual(result.normalized, "D")

    def test_accepts_runner_bold_contract(self):
        result = validate_mmlu_output("**(B)** — risposta scelta")
        self.assertTrue(result.valid)
        self.assertEqual(result.normalized, "B")

    def test_explanation_letters_do_not_override_explicit_answer(self):
        result = validate_mmlu_output("ANSWER: A. Le opzioni B, C e D sono errate.")
        self.assertTrue(result.valid)
        self.assertEqual(result.normalized, "A")

    def test_rejects_missing_answer(self):
        result = validate_mmlu_output("La spiegazione descrive il concetto ma non seleziona un'opzione.")
        self.assertFalse(result.valid)
        self.assertEqual(result.failure_code, "answer_missing")

    def test_rejects_conflicting_explicit_answers(self):
        result = validate_mmlu_output("ANSWER: A\nFinal answer: C")
        self.assertFalse(result.valid)
        self.assertEqual(result.failure_code, "answer_ambiguous")
        self.assertEqual(result.evidence["distinct_candidates"], ["A", "C"])

    def test_rejects_empty_output(self):
        result = validate_mmlu_output(None)
        self.assertFalse(result.valid)
        self.assertEqual(result.failure_code, "answer_missing")

    def test_expected_answer_only_affects_evidence(self):
        result = validate_mmlu_output("ANSWER: B", expected="C")
        self.assertTrue(result.valid)
        self.assertFalse(result.evidence["correct"])
        self.assertEqual(result.normalized, "B")


class CodingValidatorTests(unittest.TestCase):
    def test_accepts_typescript_fence_and_required_symbol(self):
        output = """Ecco l'implementazione:
```typescript
export function reverseWords(value: string): string {
  return value.trim().split(/\\s+/).reverse().join(' ');
}
```
"""
        result = validate_coding_output(output, required_symbols=["reverseWords"], min_significant_lines=3)
        self.assertTrue(result.valid)
        self.assertIn("reverseWords", result.normalized)
        self.assertEqual(result.evidence["missing_symbols"], [])

    def test_accepts_json_envelope(self):
        output = '{"language":"typescript","code":"export const add = (a: number, b: number): number => a + b;"}'
        result = validate_coding_output(output, required_symbols=["add"])
        self.assertTrue(result.valid)
        self.assertEqual(result.evidence["extraction"], "json:code")

    def test_rejects_empty_fence(self):
        result = validate_coding_output("```typescript\n\n```")
        self.assertFalse(result.valid)
        self.assertEqual(result.failure_code, "code_empty")

    def test_rejects_missing_code_block(self):
        result = validate_coding_output("La soluzione è implementata nel testo seguente, ma il codice non è incluso.")
        self.assertFalse(result.valid)
        self.assertEqual(result.failure_code, "code_missing")

    def test_rejects_wrong_language_fence(self):
        result = validate_coding_output("```python\ndef add(a, b): return a + b\n```")
        self.assertFalse(result.valid)
        self.assertEqual(result.failure_code, "code_wrong_language")

    def test_rejects_required_symbol_missing(self):
        result = validate_coding_output(
            "```ts\nexport function subtract(a: number, b: number): number { return a - b; }\n```",
            required_symbols=["add"],
        )
        self.assertFalse(result.valid)
        self.assertEqual(result.failure_code, "required_symbol_missing")
        self.assertEqual(result.evidence["missing_symbols"], ["add"])

    def test_repaired_typescript_output_passes_contract(self):
        result = validate_coding_output(
            "```typescript\nexport function add(a: number, b: number): number {\n  return a + b;\n}\n```",
            required_symbols=["add"],
            min_significant_lines=3,
        )
        self.assertTrue(result.valid)
        self.assertIsNone(result.failure_code)

    def test_retry_is_requested_for_missing_typescript_before_last_attempt(self):
        result = validate_coding_retry(
            "code_correct: implementa TypeScript",
            "La spiegazione non contiene codice.",
            is_last_attempt=False,
        )
        self.assertIsNotNone(result)
        self.assertEqual(result.failure_code, "code_missing")

    def test_retry_is_not_requested_on_last_attempt(self):
        result = validate_coding_retry(
            "code_correct: implementa TypeScript",
            "La spiegazione non contiene codice.",
            is_last_attempt=True,
        )
        self.assertIsNone(result)

    def test_retry_is_not_requested_for_non_coding_goal(self):
        result = validate_coding_retry(
            "Scrivi una spiegazione concettuale",
            "La spiegazione non contiene codice.",
            is_last_attempt=False,
        )
        self.assertIsNone(result)

    def test_retry_is_not_requested_for_valid_typescript(self):
        result = validate_coding_retry(
            "code_correct: implementa TypeScript",
            "```typescript\nexport const add = (a: number, b: number): number => a + b;\n```",
            is_last_attempt=False,
        )
        self.assertIsNone(result)

    def test_rejects_placeholder_implementation(self):
        result = validate_coding_output(
            "```typescript\nexport function add(a: number, b: number): number {\n  // TODO implement here\n  return 0;\n}\n```",
            required_symbols=["add"],
        )
        self.assertFalse(result.valid)
        self.assertEqual(result.failure_code, "code_placeholder")

    def test_rejects_non_typescript_prose_inside_fence(self):
        result = validate_coding_output("```typescript\nThis is only explanatory prose.\n```")
        self.assertFalse(result.valid)
        self.assertEqual(result.failure_code, "code_syntax_suspect")


class ReasoningValidatorTests(unittest.TestCase):
    def test_accepts_gsm8k_contract_with_thousands_separator(self):
        result = validate_reasoning_output(
            "Somma i valori: 100 + 125 = 225.\n#### 225",
            expected=225,
        )
        self.assertTrue(result.valid)
        self.assertEqual(result.normalized, "225")
        self.assertTrue(result.evidence["correct"])

    def test_accepts_labeled_final_answer(self):
        result = validate_reasoning_output("I passaggi portano al totale. Final answer: 2,250", expected=2250)
        self.assertTrue(result.valid)
        self.assertEqual(result.normalized, "2250")

    def test_classifies_wrong_numeric_answer(self):
        result = validate_reasoning_output("Calcolo completo. #### 250", expected=225)
        self.assertFalse(result.valid)
        self.assertEqual(result.failure_code, "wrong_numeric_answer")
        self.assertFalse(result.evidence["correct"])

    def test_classifies_missing_numeric_answer(self):
        result = validate_reasoning_output("La spiegazione termina senza un numero finale.", expected=225)
        self.assertFalse(result.valid)
        self.assertEqual(result.failure_code, "answer_missing")

    def test_classifies_conflicting_explicit_answers(self):
        result = validate_reasoning_output("#### 250\nFinal answer: 225", expected=225)
        self.assertFalse(result.valid)
        self.assertEqual(result.failure_code, "calculation_conflict")
        self.assertEqual(result.evidence["distinct_candidates"], [250, 225])

    def test_reasoning_retry_is_requested_for_wrong_answer_before_last_attempt(self):
        result = validate_reasoning_retry(
            "reasoning GSM8K: risolvi il problema",
            "#### 250",
            expected=225,
            is_last_attempt=False,
        )
        self.assertIsNotNone(result)
        self.assertEqual(result.failure_code, "wrong_numeric_answer")

    def test_reasoning_retry_is_not_requested_on_last_attempt(self):
        result = validate_reasoning_retry(
            "reasoning GSM8K: risolvi il problema",
            "#### 250",
            expected=225,
            is_last_attempt=True,
        )
        self.assertIsNone(result)

    def test_reasoning_retry_is_not_requested_for_non_reasoning_goal(self):
        result = validate_reasoning_retry(
            "Implementa un componente TypeScript",
            "#### 250",
            expected=225,
            is_last_attempt=False,
        )
        self.assertIsNone(result)


if __name__ == "__main__":
    unittest.main()