tests/failure_modes/test_provider_failures.py

"""
Provider Failure Cascade Tests

Test how the system handles LLM provider failures:
- All providers fail (timeout, rate limit, server error)
- Primary provider fails, fallback to secondary succeeds
- Providers fail sequentially, fallback logic verification
- No providers configured scenario

All tests use mocks to simulate provider failures without actual API calls.
"""

import pytest
import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
from typing import Dict, List


class TestAllProvidersFail:
    """Test behavior when all LLM providers fail."""

    @pytest.mark.asyncio
    async def test_all_providers_fail(self):
        """
        FAILURE MODE: All configured LLM providers fail with API errors.
        EXPECTED: Attempts all providers, returns clear error, doesn't crash.
        """
        from core.llm.byok_handler import BYOKHandler

        handler = BYOKHandler()

        # Mock all providers to fail (add them even if not configured)
        provider_ids = ["deepseek", "openai", "moonshot", "minimax"]
        for provider_id in provider_ids:
            mock_client = MagicMock()
            mock_client.chat.completions.create = AsyncMock(
                side_effect=Exception(f"{provider_id} API error (500)")
            )
            handler.clients[provider_id] = mock_client
            handler.async_clients[provider_id] = mock_client

        # Should try all providers and return error
        try:
            response = await handler.generate_response(
                prompt="test prompt",
                system_instruction="You are helpful"
            )
            # Should not crash
            assert response is not None
            # Should mention error/failure
            response_lower = response.lower()
            assert any(keyword in response_lower for keyword in ["error", "failed", "unavailable", "provider"])
        except Exception as e:
            # Exception is acceptable if it mentions providers failed
            error_str = str(e).lower()
            assert any(keyword in error_str for keyword in ["provider", "failed", "error"])

    @pytest.mark.asyncio
    async def test_all_providers_timeout(self):
        """
        FAILURE MODE: All LLM providers timeout.
        EXPECTED: Graceful degradation, timeout error message.
        """
        from core.llm.byok_handler import BYOKHandler

        handler = BYOKHandler()

        # Mock all providers to timeout (add them even if not configured)
        provider_ids = ["deepseek", "openai"]
        for provider_id in provider_ids:
            mock_client = MagicMock()
            mock_client.chat.completions.create = AsyncMock(
                side_effect=asyncio.TimeoutError(f"{provider_id} request timed out")
            )
            handler.clients[provider_id] = mock_client
            handler.async_clients[provider_id] = mock_client

        # Should timeout gracefully
        try:
            response = await handler.generate_response(
                prompt="test prompt",
                system_instruction="You are helpful"
            )
            assert response is not None
            assert "timeout" in response.lower() or "failed" in response.lower()
        except (asyncio.TimeoutError, Exception) as e:
            assert "timeout" in str(e).lower()

    @pytest.mark.asyncio
    async def test_all_providers_rate_limited(self):
        """
        FAILURE MODE: All LLM providers return rate limit errors.
        EXPECTED: Clear rate limit error message, graceful degradation.
        """
        from core.llm.byok_handler import BYOKHandler

        handler = BYOKHandler()

        # Mock all providers to rate limit (add them even if not configured)
        provider_ids = ["deepseek", "openai"]
        for provider_id in provider_ids:
            mock_client = MagicMock()
            mock_client.chat.completions.create = AsyncMock(
                side_effect=Exception(f"Rate limit exceeded (429) for {provider_id}")
            )
            handler.clients[provider_id] = mock_client
            handler.async_clients[provider_id] = mock_client

        # Should handle rate limit gracefully
        try:
            response = await handler.generate_response(
                prompt="test prompt",
                system_instruction="You are helpful"
            )
            assert response is not None
            response_lower = response.lower()
            assert any(keyword in response_lower for keyword in ["rate limit", "429", "quota", "exceeded"])
        except Exception as e:
            error_str = str(e).lower()
            assert any(keyword in error_str for keyword in ["rate limit", "429", "quota"])


class TestPrimaryProviderFailure:
    """Test fallback when primary provider fails."""

    @pytest.mark.asyncio
    async def test_primary_provider_fails_fallback_to_secondary(self):
        """
        FAILURE MODE: Primary provider fails, secondary succeeds.
        EXPECTED: Fallback to secondary provider, response returned.
        """
        from core.llm.byok_handler import BYOKHandler

        handler = BYOKHandler()

        # Mock OpenAI to fail (add even if not configured)
        mock_openai = MagicMock()
        mock_openai.chat.completions.create = AsyncMock(
            side_effect=Exception("OpenAI API error (500)")
        )
        handler.clients["openai"] = mock_openai
        handler.async_clients["openai"] = mock_openai

        # Mock Deepseek to succeed
        mock_deepseek = MagicMock()
        mock_deepseek.chat.completions.create = AsyncMock(
            return_value=MagicMock(
                choices=[MagicMock(message=MagicMock(content="Success from Deepseek"))]
            )
        )
        handler.clients["deepseek"] = mock_deepseek
        handler.async_clients["deepseek"] = mock_deepseek

        # Should fallback to Deepseek
        response = await handler.generate_response(
            prompt="test prompt",
            system_instruction="You are helpful"
        )

        # Should succeed with Deepseek response
        assert response is not None
        assert "deepseek" in response.lower() or "success" in response.lower()

    @pytest.mark.asyncio
    async def test_primary_provider_rate_limits_secondary_succeeds(self):
        """
        FAILURE MODE: Primary provider rate limited, secondary succeeds.
        EXPECTED: Fallback triggered, secondary response returned.
        """
        from core.llm.byok_handler import BYOKHandler

        handler = BYOKHandler()

        # Mock OpenAI to rate limit (add even if not configured)
        mock_openai = MagicMock()
        mock_openai.chat.completions.create = AsyncMock(
            side_effect=Exception("Rate limit exceeded (429)")
        )
        handler.clients["openai"] = mock_openai
        handler.async_clients["openai"] = mock_openai

        # Mock Deepseek to succeed
        mock_deepseek = MagicMock()
        mock_deepseek.chat.completions.create = AsyncMock(
            return_value=MagicMock(
                choices=[MagicMock(message=MagicMock(content="Response from Deepseek"))]
            )
        )
        handler.clients["deepseek"] = mock_deepseek
        handler.async_clients["deepseek"] = mock_deepseek

        # Should fallback to Deepseek
        response = await handler.generate_response(
            prompt="test prompt",
            system_instruction="You are helpful"
        )

        # Should succeed with Deepseek
        assert response is not None
        assert "deepseek" in response.lower() or "response" in response.lower()

    @pytest.mark.asyncio
    async def test_primary_provider_timeout_secondary_succeeds(self):
        """
        FAILURE MODE: Primary provider times out, secondary succeeds.
        EXPECTED: Timeout detected, secondary provider attempted, response returned.
        """
        from core.llm.byok_handler import BYOKHandler

        handler = BYOKHandler()

        # Mock OpenAI to timeout (add even if not configured)
        mock_openai = MagicMock()
        mock_openai.chat.completions.create = AsyncMock(
            side_effect=asyncio.TimeoutError("OpenAI request timed out")
        )
        handler.clients["openai"] = mock_openai
        handler.async_clients["openai"] = mock_openai

        # Mock Deepseek to succeed
        mock_deepseek = MagicMock()
        mock_deepseek.chat.completions.create = AsyncMock(
            return_value=MagicMock(
                choices=[MagicMock(message=MagicMock(content="Success from fallback"))]
            )
        )
        handler.clients["deepseek"] = mock_deepseek
        handler.async_clients["deepseek"] = mock_deepseek

        # Should fallback to Deepseek
        response = await handler.generate_response(
            prompt="test prompt",
            system_instruction="You are helpful"
        )

        # Should succeed with fallback
        assert response is not None
        assert "fallback" in response.lower() or "success" in response.lower()


class TestProviderCascade:
    """Test providers failing sequentially."""

    @pytest.mark.asyncio
    async def test_providers_fail_sequentially(self):
        """
        FAILURE MODE: Providers fail one by one, all attempted before giving up.
        EXPECTED: All providers attempted, clear error after last fails.
        """
        from core.llm.byok_handler import BYOKHandler

        handler = BYOKHandler()

        # Mock providers to fail sequentially (add even if not configured)
        provider_ids = ["deepseek", "openai", "moonshot"]
        attempted_providers = []

        async def failing_create(*args, **kwargs):
            # Track which provider was called
            # This is a simplified tracking mechanism
            raise Exception("Simulated failure")

        for provider_id in provider_ids:
            mock_client = MagicMock()
            mock_client.chat.completions.create = AsyncMock(side_effect=failing_create)
            handler.clients[provider_id] = mock_client
            handler.async_clients[provider_id] = mock_client

        # Should try all providers and fail
        try:
            response = await handler.generate_response(
                prompt="test prompt",
                system_instruction="You are helpful"
            )
            # If it returns (shouldn't happen), it should mention error
            assert response is not None
            response_lower = response.lower()
            assert any(keyword in response_lower for keyword in ["error", "failed", "provider"])
        except Exception as e:
            # Should fail with error mentioning all providers failed
            error_str = str(e).lower()
            assert any(keyword in error_str for keyword in ["provider", "failed"])

        # Track which providers were attempted
        attempted_providers = []

        # Mock providers to fail sequentially
        provider_ids = ["openai", "anthropic", "deepseek"]
        for provider_id in provider_ids:
            if provider_id not in handler.clients:
                continue

            async def mock_fail(pid=provider_id):
                attempted_providers.append(pid)
                raise Exception(f"{pid} failed")

            mock_client = MagicMock()
            mock_client.chat.completions.create = AsyncMock(side_effect=mock_fail)
            handler.clients[provider_id] = mock_client
            handler.async_clients[provider_id] = mock_client

        # Should attempt all providers
        try:
            response = await handler.generate_response(
                prompt="test prompt",
                system_instruction="You are helpful"
            )
            # If response returned, should mention all failed
            assert response is not None
        except Exception as e:
            # Exception should mention provider failure
            assert "fail" in str(e).lower() or "provider" in str(e).lower()

        # Verify multiple providers were attempted
        # (Note: This depends on BYOKHandler implementation)
        assert len(attempted_providers) >= 1

    @pytest.mark.asyncio
    async def test_provider_unavailable(self):
        """
        FAILURE MODE: Provider raises ConnectionError (unavailable).
        EXPECTED: Next provider tried, fallback works.
        """
        from core.llm.byok_handler import BYOKHandler

        handler = BYOKHandler()

        # Mock OpenAI to be unavailable
        if "openai" in handler.clients:
            mock_openai = MagicMock()
            mock_openai.chat.completions.create = AsyncMock(
                side_effect=ConnectionError("OpenAI service unavailable")
            )
            handler.clients["openai"] = mock_openai
            handler.async_clients["openai"] = mock_openai

        # Mock Anthropic to succeed
        if "anthropic" in handler.clients:
            mock_anthropic = MagicMock()
            mock_anthropic.chat.completions.create = AsyncMock(
                return_value=MagicMock(
                    choices=[MagicMock(message=MagicMock(content="Success"))]
                )
            )
            handler.clients["anthropic"] = mock_anthropic
            handler.async_clients["anthropic"] = mock_anthropic

        # Should fallback to Anthropic
        response = await handler.generate_response(
            prompt="test prompt",
            system_instruction="You are helpful"
        )

        # Should succeed with fallback
        assert response is not None

    @pytest.mark.asyncio
    async def test_provider_api_key_invalid(self):
        """
        FAILURE MODE: Provider raises AuthenticationError (invalid API key).
        EXPECTED: Fallback to next provider, auth error logged.
        """
        from core.llm.byok_handler import BYOKHandler

        handler = BYOKHandler()

        # Mock OpenAI with invalid API key
        if "openai" in handler.clients:
            mock_openai = MagicMock()
            mock_openai.chat.completions.create = AsyncMock(
                side_effect=Exception("Unauthorized (401): Invalid API key")
            )
            handler.clients["openai"] = mock_openai
            handler.async_clients["openai"] = mock_openai

        # Mock Anthropic to succeed
        if "anthropic" in handler.clients:
            mock_anthropic = MagicMock()
            mock_anthropic.chat.completions.create = AsyncMock(
                return_value=MagicMock(
                    choices=[MagicMock(message=MagicMock(content="Success"))]
                )
            )
            handler.clients["anthropic"] = mock_anthropic
            handler.async_clients["anthropic"] = mock_anthropic

        # Should fallback to Anthropic
        response = await handler.generate_response(
            prompt="test prompt",
            system_instruction="You are helpful"
        )

        # Should succeed despite auth error
        assert response is not None


class TestFallbackVerification:
    """Verify fallback provider behavior."""

    @pytest.mark.asyncio
    async def test_fallback_provider_called(self):
        """
        FAILURE MODE: Primary fails, verify fallback provider API called.
        EXPECTED: Fallback provider's API method called.
        """
        from core.llm.byok_handler import BYOKHandler

        handler = BYOKHandler()

        # Mock primary to fail
        if "openai" in handler.clients:
            mock_openai = MagicMock()
            mock_openai.chat.completions.create = AsyncMock(
                side_effect=Exception("Primary failed")
            )
            handler.clients["openai"] = mock_openai
            handler.async_clients["openai"] = mock_openai

        # Mock fallback to succeed
        if "anthropic" in handler.clients:
            mock_anthropic = MagicMock()
            mock_anthropic.chat.completions.create = AsyncMock(
                return_value=MagicMock(
                    choices=[MagicMock(message=MagicMock(content="Fallback success"))]
                )
            )
            handler.clients["anthropic"] = mock_anthropic
            handler.async_clients["anthropic"] = mock_anthropic

        # Trigger fallback
        response = await handler.generate_response(
            prompt="test prompt",
            system_instruction="You are helpful"
        )

        # Verify response
        assert response is not None

        # Note: Verifying actual method call depends on BYOKHandler implementation
        # This test documents the expected behavior

    @pytest.mark.asyncio
    async def test_no_providers_configured(self):
        """
        FAILURE MODE: No providers configured (empty providers dict).
        EXPECTED: Clear error message, graceful handling.
        """
        from core.llm.byok_handler import BYOKHandler

        handler = BYOKHandler()

        # Save original providers
        original_clients = handler.clients.copy()
        original_async_clients = handler.async_clients.copy()

        # Clear all providers
        handler.clients.clear()
        handler.async_clients.clear()

        # Should return clear error
        try:
            response = await handler.generate_response(
                prompt="test prompt",
                system_instruction="You are helpful"
            )
            # If response returned, should mention no providers
            assert response is not None
            assert "provider" in response.lower() or "configured" in response.lower()
        except Exception as e:
            # Exception should mention no providers
            assert "provider" in str(e).lower() or "configured" in str(e).lower()
        finally:
            # Restore providers
            handler.clients = original_clients
            handler.async_clients = original_async_clients


class TestProviderFailureEdgeCases:
    """Test edge cases in provider failure handling."""

    @pytest.mark.asyncio
    async def test_provider_context_window_exceeded(self):
        """
        FAILURE MODE: Provider raises context window exceeded error.
        EXPECTED: Error handled, clear message, fallback attempted.
        """
        from core.llm.byok_handler import BYOKHandler

        handler = BYOKHandler()

        # Mock OpenAI to exceed context window
        if "openai" in handler.clients:
            mock_openai = MagicMock()
            mock_openai.chat.completions.create = AsyncMock(
                side_effect=Exception("This model's maximum context length is 128000 tokens")
            )
            handler.clients["openai"] = mock_openai
            handler.async_clients["openai"] = mock_openai

        # Mock Anthropic to succeed (with larger context)
        if "anthropic" in handler.clients:
            mock_anthropic = MagicMock()
            mock_anthropic.chat.completions.create = AsyncMock(
                return_value=MagicMock(
                    choices=[MagicMock(message=MagicMock(content="Success"))]
                )
            )
            handler.clients["anthropic"] = mock_anthropic
            handler.async_clients["anthropic"] = mock_anthropic

        # Should fallback to Anthropic
        response = await handler.generate_response(
            prompt="test prompt",
            system_instruction="You are helpful"
        )

        # Should succeed with fallback
        assert response is not None

    @pytest.mark.asyncio
    async def test_provider_server_error_500(self):
        """
        FAILURE MODE: Provider returns 500 Internal Server Error.
        EXPECTED: Fallback triggered, error logged.
        """
        from core.llm.byok_handler import BYOKHandler

        handler = BYOKHandler()

        # Mock OpenAI with 500 error
        if "openai" in handler.clients:
            mock_openai = MagicMock()
            mock_openai.chat.completions.create = AsyncMock(
                side_effect=Exception("Internal server error (500)")
            )
            handler.clients["openai"] = mock_openai
            handler.async_clients["openai"] = mock_openai

        # Mock Anthropic to succeed
        if "anthropic" in handler.clients:
            mock_anthropic = MagicMock()
            mock_anthropic.chat.completions.create = AsyncMock(
                return_value=MagicMock(
                    choices=[MagicMock(message=MagicMock(content="Success"))]
                )
            )
            handler.clients["anthropic"] = mock_anthropic
            handler.async_clients["anthropic"] = mock_anthropic

        # Should fallback to Anthropic
        response = await handler.generate_response(
            prompt="test prompt",
            system_instruction="You are helpful"
        )

        # Should succeed with fallback
        assert response is not None

    @pytest.mark.asyncio
    async def test_provider_service_unavailable_503(self):
        """
        FAILURE MODE: Provider returns 503 Service Unavailable.
        EXPECTED: Fallback triggered, graceful degradation.
        """
        from core.llm.byok_handler import BYOKHandler

        handler = BYOKHandler()

        # Mock OpenAI with 503 error
        if "openai" in handler.clients:
            mock_openai = MagicMock()
            mock_openai.chat.completions.create = AsyncMock(
                side_effect=Exception("Service unavailable (503)")
            )
            handler.clients["openai"] = mock_openai
            handler.async_clients["openai"] = mock_openai

        # Mock Anthropic to succeed
        if "anthropic" in handler.clients:
            mock_anthropic = MagicMock()
            mock_anthropic.chat.completions.create = AsyncMock(
                return_value=MagicMock(
                    choices=[MagicMock(message=MagicMock(content="Success"))]
                )
            )
            handler.clients["anthropic"] = mock_anthropic
            handler.async_clients["anthropic"] = mock_anthropic

        # Should fallback to Anthropic
        response = await handler.generate_response(
            prompt="test prompt",
            system_instruction="You are helpful"
        )

        # Should succeed with fallback
        assert response is not None