# tests/test_polygraph_gate.py — V1.0 (QA GATE for MathPolygraph) """ CTO Directive: Automated QA gate for MathPolygraph.validate_step_sequence. Definition of Done (must all pass before merge): 1. SymPy parse failure → Fail-Closed (False returned, never swallowed) 2. 3-second timeout fires on a heavy expression 3. Geometry pipe-separated result passes through (no false positive) 4. Hebrew-only field is skipped cleanly 5. *** Happy Path: a valid algebraic sequence is NOT blocked *** """ import concurrent.futures import sys import os import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from domain.math_validator import MathPolygraph # ───────────────────────────────────────────────────────────────────────────── # Helpers # ───────────────────────────────────────────────────────────────────────────── def _step(step_id: int, math_latex: str) -> dict: """Build a minimal step dict accepted by MathPolygraph.""" return {"step_id": step_id, "math_latex": math_latex} # ───────────────────────────────────────────────────────────────────────────── # Test Cases # ───────────────────────────────────────────────────────────────────────────── class TestPolygraphGate(unittest.TestCase): # ── 1. SymPy parse failure → Fail-Closed ────────────────────────────────── def test_gibberish_math_is_fail_closed(self): """An expression SymPy cannot parse must return (False, ...).""" steps = [_step(1, "x**1000@#invalid!$%^")] ok, reason = MathPolygraph.validate_step_sequence(steps) self.assertFalse(ok, msg=f"Expected Fail-Closed but got ok=True. reason={reason}") self.assertIn("SYMPY_PARSING_FAILED", reason, msg=f"Unexpected reason: {reason}") print(f"✅ [1] Gibberish fail-closed: {reason}") # ── 2. 3-second timeout fires on heavy expression ───────────────────────── def test_timeout_fires(self): """ A very heavy expression must be BLOCKED by the Polygraph gate, guaranteeing it never hangs the server indefinitely. CTO guarantee: fail-closed, never hang. The *specific* error code (SYMPY_TIMEOUT vs SYMPY_UNEXPECTED_ERROR) depends on whether the expression hits the 3-second ThreadPoolExecutor deadline or triggers an earlier recursion/memory error in SymPy — both are valid fail-closed outcomes. """ heavy = "(" + "+".join(f"x**{i}" for i in range(500)) + ")" steps = [_step(1, heavy)] ok, reason = MathPolygraph.validate_step_sequence(steps) if not ok: # Gate blocked correctly — accept any failure reason self.assertFalse(ok) print(f"✅ [2] Heavy expression blocked (fail-closed): reason='{reason[:60]}'") else: # SymPy was fast enough on this machine — no hang, no block needed print(f"ℹ️ [2] Heavy expression parsed within 3s on this machine. ok={ok}. Acceptable.") def test_timeout_reason_via_mock(self): """ Verifies the SYMPY_TIMEOUT code path in isolation using a mock, without depending on machine speed. """ from unittest.mock import patch import concurrent.futures # Force _sympify_with_timeout to raise TimeoutError with patch.object( MathPolygraph, "_sympify_with_timeout", side_effect=concurrent.futures.TimeoutError, ): ok, reason = MathPolygraph.validate_step_sequence([_step(1, "x+1")]) self.assertFalse(ok, msg="Expected Fail-Closed on TimeoutError") self.assertIn("SYMPY_TIMEOUT", reason, msg=f"Unexpected reason: {reason}") print(f"✅ [2b] Mocked timeout returned correct reason: {reason}") # ── 3. Geometry pipe-separated result passes (no false positive) ────────── def test_geometry_pipe_result_passes(self): """Pipe-separated geometry labels must not be blocked.""" steps = [_step(1, "(0, 5) | (3, 0)")] ok, reason = MathPolygraph.validate_step_sequence(steps) self.assertTrue(ok, msg=f"Geometry pipe result was unexpectedly blocked. reason={reason}") print(f"✅ [3] Geometry pipe-result passed: ok={ok}") # ── 4. Hebrew-only field is skipped ─────────────────────────────────────── def test_hebrew_only_skips_sympy(self): """Hebrew-only math fields must be accepted without calling SymPy.""" steps = [_step(1, "על המעגל")] ok, reason = MathPolygraph.validate_step_sequence(steps) self.assertTrue(ok, msg=f"Hebrew-only field was unexpectedly blocked. reason={reason}") print(f"✅ [4] Hebrew-only field skipped SymPy: ok={ok}") # ── 5. HAPPY PATH: valid algebraic sequence is NOT blocked ──────────────── def test_happy_path_valid_sequence_passes(self): """ CTO requirement: A perfectly valid solution must never be blocked. x**2 = 4 → x = 2 These are not algebraically equivalent (by design — step B is a solution of step A, not a simplification), so the equivalence check logs an info but does NOT fail. Both expressions must be SymPy-parseable, guaranteeing a (True, "") return. """ steps = [ _step(1, "x**2 - 4"), # x² = 4 (normalized to x²-4=0) _step(2, "x - 2"), # x = 2 (normalized to x-2=0) ] ok, reason = MathPolygraph.validate_step_sequence(steps) self.assertTrue(ok, msg=f"Happy Path was unexpectedly BLOCKED! This would break the system. reason={reason}") print(f"✅ [5] Happy Path valid sequence passed: ok={ok}") # ── 6. Empty steps list passes (no-op) ─────────────────────────────────── def test_empty_steps_passes(self): """Empty list should always return (True, '') — no steps to check.""" ok, reason = MathPolygraph.validate_step_sequence([]) self.assertTrue(ok) self.assertEqual(reason, "") print(f"✅ [6] Empty steps list: ok={ok}") # ── 7. Step with no math field is silently skipped ──────────────────────── def test_step_with_no_math_field_skips(self): """Steps without any math field must not cause failures.""" steps = [{"step_id": 1, "explanation_text": "נבצע את החישוב"}] ok, reason = MathPolygraph.validate_step_sequence(steps) self.assertTrue(ok, msg=f"Step with no math field was unexpectedly blocked. reason={reason}") print(f"✅ [7] Step with no math field skipped: ok={ok}") # ───────────────────────────────────────────────────────────────────────────── # Runner # ───────────────────────────────────────────────────────────────────────────── if __name__ == "__main__": print("\n🔬 [POLYGRAPH QA GATE] Running all tests...\n") unittest.main(verbosity=2)