""" Critical Error Path Test Configuration Shared fixtures and utilities for testing critical error paths that are rare in normal operation but essential for production reliability. These tests focus on database connection failures with retry logic validation. Critical paths tested: - Database connection failures (connection refused, timeout, pool exhausted) - Connection pool exhaustion and recovery - Database deadlock handling with retry logic - Connection timeout scenarios - Error propagation from database to service layer to API These fixtures extend patterns from backend/tests/error_paths/conftest.py and backend/tests/failure_modes/conftest.py, specializing them for critical path testing. """ import pytest import logging from unittest.mock import patch, MagicMock, AsyncMock, call from unittest.mock import _patch as _patch_type from typing import Any, Dict, List, Optional, Type, Callable, Generator from contextlib import contextmanager from sqlalchemy.exc import IntegrityError, OperationalError, ProgrammingError, DataError, DBAPIError from sqlalchemy.orm import Session from sqlalchemy.pool import Pool, QueuePool import time logger = logging.getLogger(__name__) # ============================================================================ # Database Connection Failure Fixtures # ============================================================================ @pytest.fixture def db_session_with_retry(): """ Database session that tracks retry attempts for testing retry logic. Usage: def test_connection_retry(db_session_with_retry): with db_session_with_retry() as (db, retry_tracker): # Perform database operation that may fail # retry_tracker tracks retry attempts assert retry_tracker['retry_count'] > 0 Returns: Context manager that yields (db_session, retry_tracker) retry_tracker dict contains: - retry_count: Number of retry attempts - last_error: Last exception raised - retry_delay: Total delay from retries """ @contextmanager def _session_context(): from core.database import SessionLocal retry_tracker = { 'retry_count': 0, 'last_error': None, 'retry_delay': 0.0, 'retry_timestamps': [] } # Track retry attempts by patching SessionLocal original_connect = SessionLocal().connection def tracked_connect(): retry_tracker['retry_count'] += 1 retry_tracker['retry_timestamps'].append(time.time()) return original_connect() try: db = SessionLocal() yield db, retry_tracker except Exception as e: retry_tracker['last_error'] = e raise finally: if 'db' in locals(): db.close() return _session_context() @pytest.fixture def mock_connection_failure(): """ Patch engine.connect to raise OperationalError (connection failure). Usage: def test_connection_refused_triggers_retry(mock_connection_failure): with mock_connection_failure(fail_count=2) as mock_connect: # First 2 calls fail, 3rd succeeds # Test that retry logic handles connection failures Args: fail_count: Number of times to raise error before succeeding (default: 1) error_message: Custom error message (default: "Connection refused") error_type: Type of OperationalError (default: "connection_refused") Returns: Context manager that patches engine.connect """ @contextmanager def _patch(fail_count: int = 1, error_message: str = "Connection refused", error_type: str = "connection_refused"): call_count = [0] error_messages = { "connection_refused": "could not connect to server: Connection refused", "timeout": "server closed the connection unexpectedly", "network_unreachable": "Network is unreachable", "host_not_found": "could not translate host name", } message = error_messages.get(error_type, error_message) def side_effect(*args, **kwargs): call_count[0] += 1 if call_count[0] <= fail_count: raise OperationalError(message, {}, None) # Return mock connection after failures exhausted mock_conn = MagicMock() mock_conn.connect.return_value = MagicMock() return mock_conn with patch("core.database.engine.connect", side_effect=side_effect): yield return _patch @pytest.fixture def mock_pool_exhaustion(): """ Simulate connection pool exhaustion with OperationalError. Usage: def test_pool_exhaustion_handling(mock_pool_exhaustion): with mock_pool_exhaustion(): # Test pool exhausted error handling Args: pool_size: Current pool size (default: 20) max_overflow: Max overflow (default: 30) error_message: Custom error message (default: "connection pool exhausted") Returns: Context manager that patches pool checkout """ @contextmanager def _patch(pool_size: int = 20, max_overflow: int = 30, error_message: str = "connection pool exhausted"): def side_effect(*args, **kwargs): raise OperationalError( f"connection pool exhausted: {pool_size + max_overflow} connections already in use", {}, None ) with patch("sqlalchemy.pool.QueuePool.connect", side_effect=side_effect): yield return _patch @pytest.fixture def mock_deadlock_scenario(): """ Mock deadlock OperationalError with retry capability. Usage: def test_deadlock_triggers_retry(mock_deadlock_scenario): with mock_deadlock_scenario(retry_count=3) as deadlock_mock: # First 3 attempts deadlock, 4th succeeds # Test deadlock retry logic Args: retry_count: Number of deadlocks before success (default: 1) error_message: Custom error message (default: "deadlock detected") Returns: Context manager that patches commit to raise deadlock error """ @contextmanager def _patch(retry_count: int = 1, error_message: str = "deadlock detected"): call_count = [0] def side_effect(*args, **kwargs): call_count[0] += 1 if call_count[0] <= retry_count: # Deadlock error with PostgreSQL-specific message raise OperationalError( f"deadlock detected: {error_message}", {}, Exception("DETAIL: Process 123 waits for ShareLock on transaction 456; blocked by process 789.") ) return None # Success after retries with patch("sqlalchemy.orm.Session.commit", side_effect=side_effect): yield return _patch @pytest.fixture def track_retry_attempts(): """ Track and verify retry attempts during database operations. Usage: def test_retry_logic(track_retry_attempts): with track_retry_attempts() as retry_tracker: # Perform operation that may retry assert retry_tracker.call_count > 0 assert retry_tracker.total_delay > 0 Returns: Context manager that yields RetryTracker object with: - call_count: Number of retry attempts - total_delay: Total time spent in retries - call_timestamps: List of retry timestamps - exceptions: List of exceptions caught """ class RetryTracker: def __init__(self): self.call_count = 0 self.total_delay = 0.0 self.call_timestamps = [] self.exceptions = [] def record_call(self, exception: Exception = None): self.call_count += 1 self.call_timestamps.append(time.time()) if exception: self.exceptions.append(exception) def get_delay(self) -> float: """Calculate total delay between retries""" if len(self.call_timestamps) < 2: return 0.0 return self.call_timestamps[-1] - self.call_timestamps[0] def verify_retry_count(self, expected: int) -> bool: """Verify exact retry count""" return self.call_count == expected def verify_min_retries(self, min_count: int) -> bool: """Verify minimum retry count""" return self.call_count >= min_count def verify_exponential_backoff(self) -> bool: """Verify delays increase exponentially (roughly)""" if len(self.call_timestamps) < 3: return True # Not enough data points delays = [] for i in range(1, len(self.call_timestamps)): delays.append(self.call_timestamps[i] - self.call_timestamps[i-1]) # Check if delays are generally increasing return all(delays[i] * 0.8 <= delays[i+1] for i in range(len(delays)-1)) @contextmanager def _track(): tracker = RetryTracker() try: yield tracker finally: logger.info(f"Retry tracker: {tracker.call_count} attempts, {tracker.get_delay():.2f}s total delay") return _track # ============================================================================ # Connection Timeout Fixtures # ============================================================================ @pytest.fixture def mock_connection_timeout(): """ Mock database connection timeout with OperationalError. Usage: def test_connection_timeout_handling(mock_connection_timeout): with mock_connection_timeout(timeout_seconds=30): # Test timeout handling Args: timeout_seconds: Connection timeout duration (default: 30) error_type: Type of timeout (default: "connect") Returns: Context manager that patches connection with timeout error """ @contextmanager def _patch(timeout_seconds: int = 30, error_type: str = "connect"): error_messages = { "connect": f"server closed the connection unexpectedly: This probably means the server terminated abnormally before or while processing the request. (timeout after {timeout_seconds}s)", "query": f"canceling statement due to statement timeout: timeout after {timeout_seconds}s", "lock": "canceling statement due to lock timeout", } message = error_messages.get(error_type, f"timeout after {timeout_seconds}s") def side_effect(*args, **kwargs): raise OperationalError(message, {}, None) with patch("core.database.engine.connect", side_effect=side_effect): yield return _patch @pytest.fixture def mock_query_timeout(): """ Mock query execution timeout with OperationalError. Usage: def test_query_timeout_handling(mock_query_timeout): with mock_query_timeout(timeout_ms=5000): # Test query timeout handling Args: timeout_ms: Query timeout in milliseconds (default: 5000) recover_after: Number of timeouts before recovery (default: 1) Returns: Context manager that patches execute with timeout error """ @contextmanager def _patch(timeout_ms: int = 5000, recover_after: int = 1): call_count = [0] def side_effect(*args, **kwargs): call_count[0] += 1 if call_count[0] <= recover_after: raise OperationalError( f"canceling statement due to statement timeout: timeout after {timeout_ms}ms", {}, None ) # Return mock result after recovery return MagicMock() with patch("sqlalchemy.orm.Session.execute", side_effect=side_effect): yield return _patch # ============================================================================ # Error Propagation Fixtures # ============================================================================ @pytest.fixture def verify_error_propagation(): """ Verify error propagation from database through service layer to API. Usage: def test_error_propagation(verify_error_propagation): with verify_error_propagation() as verifier: # Trigger database error raise OperationalError("test", {}, None) # verifier captures exception chain Returns: Context manager that yields ErrorPropagationVerifier with: - original_exception: Original database exception - wrapped_exceptions: List of wrapped exceptions - final_exception: Final exception raised to caller - propagation_chain: Full exception chain """ class ErrorPropagationVerifier: def __init__(self): self.original_exception = None self.wrapped_exceptions = [] self.final_exception = None self.propagation_chain = [] def record_exception(self, exc: Exception): """Record exception in propagation chain""" if self.original_exception is None: self.original_exception = exc self.wrapped_exceptions.append(exc) self.final_exception = exc self.propagation_chain.append({ 'type': type(exc).__name__, 'message': str(exc), 'module': type(exc).__module__ }) def verify_exception_type(self, expected_type: Type[Exception]) -> bool: """Verify final exception is expected type""" return isinstance(self.final_exception, expected_type) def verify_error_code(self, expected_code: str) -> bool: """Verify AtomException error code""" if hasattr(self.final_exception, 'error_code'): return str(self.final_exception.error_code) == expected_code return False def verify_chain_contains(self, exception_type: Type[Exception]) -> bool: """Verify exception chain contains specific type""" for exc_info in self.propagation_chain: if exc_info['type'] == exception_type.__name__: return True return False @contextmanager def _verify(): verifier = ErrorPropagationVerifier() try: yield verifier except Exception as e: verifier.record_exception(e) raise finally: logger.info(f"Error propagation chain: {verifier.propagation_chain}") return _verify # ============================================================================ # Retry Logic Verification Helpers # ============================================================================ @pytest.fixture def assert_retry_occurred(): """ Verify that retry logic was triggered during operation. Usage: def test_connection_retry(assert_retry_occurred): with track_retry_attempts() as tracker: # Operation that should retry pass assert_retry_occurred(tracker, min_retries=1) Args: tracker: RetryTracker instance min_retries: Minimum expected retry count (default: 1) """ def _verify(tracker, min_retries: int = 1): assert tracker.call_count >= min_retries, f"Expected at least {min_retries} retries, got {tracker.call_count}" logger.info(f"Retry verified: {tracker.call_count} attempts") return True return _verify @pytest.fixture def assert_exponential_backoff(): """ Verify retry delays follow exponential backoff pattern. Usage: def test_exponential_backoff(assert_exponential_backoff): with track_retry_attempts() as tracker: # Operation with retries pass assert_exponential_backoff(tracker, base_delay=1.0) Args: tracker: RetryTracker instance base_delay: Base delay in seconds (default: 1.0) tolerance: Tolerance for backoff verification (default: 0.2) """ def _verify(tracker, base_delay: float = 1.0, tolerance: float = 0.2): if len(tracker.call_timestamps) < 3: logger.warning(f"Not enough retry attempts to verify exponential backoff: {tracker.call_count}") return True delays = [] for i in range(1, len(tracker.call_timestamps)): delays.append(tracker.call_timestamps[i] - tracker.call_timestamps[i-1]) # Expected delays: base_delay, 2*base_delay, 4*base_delay, ... for i, delay in enumerate(delays): expected = base_delay * (2 ** i) if not (expected * (1 - tolerance) <= delay <= expected * (1 + tolerance)): logger.warning(f"Retry delay {i}: expected {expected:.2f}s, got {delay:.2f}s") logger.info(f"Exponential backoff verified: {delays}") return True return _verify @pytest.fixture def assert_max_retries_not_exceeded(): """ Verify retry logic respects maximum retry limit. Usage: def test_max_retries(assert_max_retries_not_exceeded): with track_retry_attempts() as tracker: # Operation with limited retries pass assert_max_retries_not_exceeded(tracker, max_retries=3) Args: tracker: RetryTracker instance max_retries: Maximum allowed retries (default: 3) """ def _verify(tracker, max_retries: int = 3): assert tracker.call_count <= max_retries, f"Retry count {tracker.call_count} exceeds max {max_retries}" logger.info(f"Max retries verified: {tracker.call_count} <= {max_retries}") return True return _verify # ============================================================================ # Database Session Helpers # ============================================================================ @pytest.fixture def mock_session_with_retry_decorator(): """ Mock a service function that uses @retry_with_backoff decorator. Usage: def test_retry_decorator(mock_session_with_retry_decorator): with mock_session_with_retry_decorator(max_retries=3) as mock_func: # Test decorated function retry behavior Args: max_retries: Maximum retry attempts (default: 3) backoff_base: Base backoff delay in seconds (default: 1.0) Returns: Context manager that mocks decorated function """ @contextmanager def _mock(max_retries: int = 3, backoff_base: float = 1.0): call_count = [0] delays = [] def mock_func_with_retry(*args, **kwargs): call_count[0] += 1 if call_count[0] < max_retries: delays.append(backoff_base * (2 ** (call_count[0] - 1))) raise OperationalError("Temporary failure", {}, None) # Success on final attempt return MagicMock() with patch("core.auto_healing.retry_with_backoff", return_value=mock_func_with_retry): yield mock_func_with_retry return _mock # ============================================================================ # Test Data Generators # ============================================================================ @pytest.fixture def connection_error_scenarios(): """ Generate various connection error scenarios for testing. Returns: List of (error_type, error_message, recoverable) tuples """ return [ ("connection_refused", "could not connect to server: Connection refused", True), ("timeout", "server closed the connection unexpectedly", True), ("network_unreachable", "Network is unreachable", False), ("host_not_found", "could not translate host name", False), ("authentication_failed", "password authentication failed", False), ("ssl_error", "SSL error: certificate verify failed", False), ] @pytest.fixture def deadlock_scenarios(): """ Generate various deadlock scenarios for testing. Returns: List of (scenario_type, error_message, retry_success) tuples """ return [ ("simple_deadlock", "deadlock detected", True), ("lock_timeout", "canceling statement due to lock timeout", True), ("serialization_failure", "could not serialize access due to concurrent update", True), ] @pytest.fixture def pool_exhaustion_scenarios(): """ Generate various pool exhaustion scenarios for testing. Returns: List of (scenario_type, pool_size, max_overflow) tuples """ return [ ("small_pool", 5, 5), ("medium_pool", 20, 30), ("large_pool", 50, 50), ("exhausted_with_overflow", 20, 0), ] # ============================================================================ # Verification Helpers # ============================================================================ @pytest.fixture def verify_database_session_health(): """ Verify database session is healthy after error recovery. Usage: def test_session_recovery(verify_database_session_health): with get_db_session() as db: # Trigger and recover from error pass verify_database_session_health(db) Args: db: Database session to verify Returns: True if session is healthy, raises AssertionError otherwise """ def _verify(db: Session): # Try a simple query to verify session health try: result = db.execute("SELECT 1").scalar() assert result == 1, "Session health check failed" logger.info("Database session health verified") return True except Exception as e: raise AssertionError(f"Database session unhealthy: {e}") return _verify