Spaces:
Running
Running
File size: 9,639 Bytes
ed28aa2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 | import unittest
from benchmarks.validators import (
validate_coding_output,
validate_coding_retry,
validate_mmlu_output,
validate_reasoning_output,
validate_reasoning_retry,
)
class MMLUValidatorTests(unittest.TestCase):
def test_accepts_explicit_answer_with_explanation(self):
result = validate_mmlu_output(
"ANSWER: C\nPerché la complessità nel caso peggiore è quadratica.",
expected="C",
)
self.assertTrue(result.valid)
self.assertEqual(result.normalized, "C")
self.assertTrue(result.evidence["correct"])
def test_accepts_marked_choice(self):
result = validate_mmlu_output("La scelta corretta è (B). La stack segue LIFO.")
self.assertTrue(result.valid)
self.assertEqual(result.normalized, "B")
def test_accepts_single_isolated_letter(self):
result = validate_mmlu_output("D")
self.assertTrue(result.valid)
self.assertEqual(result.normalized, "D")
def test_accepts_final_answer_contract_used_by_retry(self):
result = validate_mmlu_output("Final answer: D\nThe two values overflow because both are negative.")
self.assertTrue(result.valid)
self.assertEqual(result.normalized, "D")
def test_accepts_runner_bold_contract(self):
result = validate_mmlu_output("**(B)** — risposta scelta")
self.assertTrue(result.valid)
self.assertEqual(result.normalized, "B")
def test_explanation_letters_do_not_override_explicit_answer(self):
result = validate_mmlu_output("ANSWER: A. Le opzioni B, C e D sono errate.")
self.assertTrue(result.valid)
self.assertEqual(result.normalized, "A")
def test_rejects_missing_answer(self):
result = validate_mmlu_output("La spiegazione descrive il concetto ma non seleziona un'opzione.")
self.assertFalse(result.valid)
self.assertEqual(result.failure_code, "answer_missing")
def test_rejects_conflicting_explicit_answers(self):
result = validate_mmlu_output("ANSWER: A\nFinal answer: C")
self.assertFalse(result.valid)
self.assertEqual(result.failure_code, "answer_ambiguous")
self.assertEqual(result.evidence["distinct_candidates"], ["A", "C"])
def test_rejects_empty_output(self):
result = validate_mmlu_output(None)
self.assertFalse(result.valid)
self.assertEqual(result.failure_code, "answer_missing")
def test_expected_answer_only_affects_evidence(self):
result = validate_mmlu_output("ANSWER: B", expected="C")
self.assertTrue(result.valid)
self.assertFalse(result.evidence["correct"])
self.assertEqual(result.normalized, "B")
class CodingValidatorTests(unittest.TestCase):
def test_accepts_typescript_fence_and_required_symbol(self):
output = """Ecco l'implementazione:
```typescript
export function reverseWords(value: string): string {
return value.trim().split(/\\s+/).reverse().join(' ');
}
```
"""
result = validate_coding_output(output, required_symbols=["reverseWords"], min_significant_lines=3)
self.assertTrue(result.valid)
self.assertIn("reverseWords", result.normalized)
self.assertEqual(result.evidence["missing_symbols"], [])
def test_accepts_json_envelope(self):
output = '{"language":"typescript","code":"export const add = (a: number, b: number): number => a + b;"}'
result = validate_coding_output(output, required_symbols=["add"])
self.assertTrue(result.valid)
self.assertEqual(result.evidence["extraction"], "json:code")
def test_rejects_empty_fence(self):
result = validate_coding_output("```typescript\n\n```")
self.assertFalse(result.valid)
self.assertEqual(result.failure_code, "code_empty")
def test_rejects_missing_code_block(self):
result = validate_coding_output("La soluzione è implementata nel testo seguente, ma il codice non è incluso.")
self.assertFalse(result.valid)
self.assertEqual(result.failure_code, "code_missing")
def test_rejects_wrong_language_fence(self):
result = validate_coding_output("```python\ndef add(a, b): return a + b\n```")
self.assertFalse(result.valid)
self.assertEqual(result.failure_code, "code_wrong_language")
def test_rejects_required_symbol_missing(self):
result = validate_coding_output(
"```ts\nexport function subtract(a: number, b: number): number { return a - b; }\n```",
required_symbols=["add"],
)
self.assertFalse(result.valid)
self.assertEqual(result.failure_code, "required_symbol_missing")
self.assertEqual(result.evidence["missing_symbols"], ["add"])
def test_repaired_typescript_output_passes_contract(self):
result = validate_coding_output(
"```typescript\nexport function add(a: number, b: number): number {\n return a + b;\n}\n```",
required_symbols=["add"],
min_significant_lines=3,
)
self.assertTrue(result.valid)
self.assertIsNone(result.failure_code)
def test_retry_is_requested_for_missing_typescript_before_last_attempt(self):
result = validate_coding_retry(
"code_correct: implementa TypeScript",
"La spiegazione non contiene codice.",
is_last_attempt=False,
)
self.assertIsNotNone(result)
self.assertEqual(result.failure_code, "code_missing")
def test_retry_is_not_requested_on_last_attempt(self):
result = validate_coding_retry(
"code_correct: implementa TypeScript",
"La spiegazione non contiene codice.",
is_last_attempt=True,
)
self.assertIsNone(result)
def test_retry_is_not_requested_for_non_coding_goal(self):
result = validate_coding_retry(
"Scrivi una spiegazione concettuale",
"La spiegazione non contiene codice.",
is_last_attempt=False,
)
self.assertIsNone(result)
def test_retry_is_not_requested_for_valid_typescript(self):
result = validate_coding_retry(
"code_correct: implementa TypeScript",
"```typescript\nexport const add = (a: number, b: number): number => a + b;\n```",
is_last_attempt=False,
)
self.assertIsNone(result)
def test_rejects_placeholder_implementation(self):
result = validate_coding_output(
"```typescript\nexport function add(a: number, b: number): number {\n // TODO implement here\n return 0;\n}\n```",
required_symbols=["add"],
)
self.assertFalse(result.valid)
self.assertEqual(result.failure_code, "code_placeholder")
def test_rejects_non_typescript_prose_inside_fence(self):
result = validate_coding_output("```typescript\nThis is only explanatory prose.\n```")
self.assertFalse(result.valid)
self.assertEqual(result.failure_code, "code_syntax_suspect")
class ReasoningValidatorTests(unittest.TestCase):
def test_accepts_gsm8k_contract_with_thousands_separator(self):
result = validate_reasoning_output(
"Somma i valori: 100 + 125 = 225.\n#### 225",
expected=225,
)
self.assertTrue(result.valid)
self.assertEqual(result.normalized, "225")
self.assertTrue(result.evidence["correct"])
def test_accepts_labeled_final_answer(self):
result = validate_reasoning_output("I passaggi portano al totale. Final answer: 2,250", expected=2250)
self.assertTrue(result.valid)
self.assertEqual(result.normalized, "2250")
def test_classifies_wrong_numeric_answer(self):
result = validate_reasoning_output("Calcolo completo. #### 250", expected=225)
self.assertFalse(result.valid)
self.assertEqual(result.failure_code, "wrong_numeric_answer")
self.assertFalse(result.evidence["correct"])
def test_classifies_missing_numeric_answer(self):
result = validate_reasoning_output("La spiegazione termina senza un numero finale.", expected=225)
self.assertFalse(result.valid)
self.assertEqual(result.failure_code, "answer_missing")
def test_classifies_conflicting_explicit_answers(self):
result = validate_reasoning_output("#### 250\nFinal answer: 225", expected=225)
self.assertFalse(result.valid)
self.assertEqual(result.failure_code, "calculation_conflict")
self.assertEqual(result.evidence["distinct_candidates"], [250, 225])
def test_reasoning_retry_is_requested_for_wrong_answer_before_last_attempt(self):
result = validate_reasoning_retry(
"reasoning GSM8K: risolvi il problema",
"#### 250",
expected=225,
is_last_attempt=False,
)
self.assertIsNotNone(result)
self.assertEqual(result.failure_code, "wrong_numeric_answer")
def test_reasoning_retry_is_not_requested_on_last_attempt(self):
result = validate_reasoning_retry(
"reasoning GSM8K: risolvi il problema",
"#### 250",
expected=225,
is_last_attempt=True,
)
self.assertIsNone(result)
def test_reasoning_retry_is_not_requested_for_non_reasoning_goal(self):
result = validate_reasoning_retry(
"Implementa un componente TypeScript",
"#### 250",
expected=225,
is_last_attempt=False,
)
self.assertIsNone(result)
if __name__ == "__main__":
unittest.main()
|