""" Tests for OpenRouter JSON parsing reliability and model-chain retry logic. All tests run against mock provider by default. Tests that exercise OpenRouterAIProvider internals monkey-patch the HTTP client so no real API calls are made. """ from __future__ import annotations import json import os import sys from pathlib import Path from unittest.mock import MagicMock, patch import pytest BACKEND_DIR = Path(__file__).resolve().parents[1] if str(BACKEND_DIR) not in sys.path: sys.path.insert(0, str(BACKEND_DIR)) os.environ.setdefault("AI_PROVIDER", "mock") os.environ.setdefault("AUTH_ENABLED", "false") from app.core.config import get_settings get_settings.cache_clear() from app.services.ai_provider import ( AIProviderError, _FatalAPIError, _http_status_code, _parse_json_text, _validate_or_pass, MockAIProvider, OpenRouterAIProvider, SimpleExplanationAIOutput, ) # --------------------------------------------------------------------------- # _parse_json_text unit tests # --------------------------------------------------------------------------- def test_parse_plain_json(): payload = {"simple_meaning": "test", "exam_answer": "ok"} assert _parse_json_text(json.dumps(payload)) == payload def test_parse_markdown_fenced_json(): payload = {"simple_meaning": "photosynthesis"} raw = f"```json\n{json.dumps(payload)}\n```" assert _parse_json_text(raw) == payload def test_parse_fenced_json_no_language_tag(): payload = {"title": "notes"} raw = f"```\n{json.dumps(payload)}\n```" assert _parse_json_text(raw) == payload def test_parse_json_with_prose_prefix(): """Model outputs extra text before the JSON object.""" payload = {"simple_meaning": "hello"} raw = f"Sure! Here is the result:\n{json.dumps(payload)}\nHope that helps." result = _parse_json_text(raw) assert result == payload def test_parse_json_with_prose_both_sides(): payload = {"title": "EM Induction"} raw = f"Here you go:\n{json.dumps(payload)}\n\nLet me know if you need more." assert _parse_json_text(raw) == payload def test_parse_invalid_json_raises(): with pytest.raises(AIProviderError): _parse_json_text("This is not JSON at all") def test_parse_json_array_raises(): """JSON array is not an object — should raise.""" with pytest.raises(AIProviderError): _parse_json_text(json.dumps([1, 2, 3])) def test_parse_empty_string_raises(): with pytest.raises(AIProviderError): _parse_json_text("") # --------------------------------------------------------------------------- # _validate_or_pass unit tests # --------------------------------------------------------------------------- def test_validate_good_data_returns_model_dump(): data = { "simple_meaning": "Plants make food", "explain_like_15_year_old": "Like a kitchen for sunlight", "real_life_example": "Solar panels", "memory_trick": "SUN = SUNtain", "exam_answer": "Photosynthesis: green plants...", } result = _validate_or_pass(data, SimpleExplanationAIOutput) assert result["simple_meaning"] == "Plants make food" # Pydantic should fill in missing list fields with defaults assert isinstance(result["step_by_step"], list) def test_validate_bad_data_returns_raw(): """Validation failure should return raw dict, not crash.""" raw = {"garbage_field": 999} result = _validate_or_pass(raw, SimpleExplanationAIOutput) # Should return something (either validated defaults or raw data) assert isinstance(result, dict) # --------------------------------------------------------------------------- # _http_status_code helper # --------------------------------------------------------------------------- def test_http_status_code_from_status_attribute(): exc = Exception("bad") exc.status_code = 429 # type: ignore[attr-defined] assert _http_status_code(exc) == 429 def test_http_status_code_missing_returns_none(): assert _http_status_code(ValueError("no code")) is None # --------------------------------------------------------------------------- # OpenRouterAIProvider model-chain retry logic (mocked) # --------------------------------------------------------------------------- def _make_provider(api_key: str = "test-key") -> OpenRouterAIProvider: """Create an OpenRouterAIProvider without calling __init__. Uses __new__ to bypass __init__ (which would try to import openai and create a real HTTP client). All attributes are set manually so the provider is fully functional for unit tests that only touch logic. """ os.environ["OPENROUTER_API_KEY"] = api_key get_settings.cache_clear() # __new__ skips __init__ — no real OpenAI client is created. provider = OpenRouterAIProvider.__new__(OpenRouterAIProvider) provider._settings = get_settings() provider._fallback = MockAIProvider() provider._models = { "main": "deepseek/deepseek-v4-flash:free", "llama": "meta-llama/llama-3.3-70b-instruct:free", "gpt_oss": "openai/gpt-oss-120b:free", "nemotron": "nvidia/nemotron-3-nano-30b-a3b:free", } provider._client = MagicMock() provider.model_name = "openrouter:deepseek/deepseek-v4-flash:free" provider.is_fallback = False provider.fallback_reason = None provider.last_error_code = None return provider def _good_completion(content: str) -> MagicMock: choice = MagicMock() choice.message.content = content resp = MagicMock() resp.choices = [choice] return resp def test_model_chain_retries_on_bad_json(): """First model returns garbage JSON → second model succeeds.""" provider = _make_provider() good_payload = json.dumps({ "simple_meaning": "Plants make food using sunlight", "explain_like_15_year_old": "Like a food factory", "real_life_example": "Solar energy", "memory_trick": "SUN", "exam_answer": "Photosynthesis is...", }) provider._client.chat.completions.create.side_effect = [ _good_completion("NOT JSON AT ALL !!!"), # first model fails json parse _good_completion(good_payload), # second model succeeds ] result = provider._generate_json( task="explain", context="photosynthesis", language="English", metadata=None, response_schema=SimpleExplanationAIOutput, route=("main", "llama"), ) assert "simple_meaning" in result assert provider.model_name.startswith("openrouter:") def test_all_models_fail_raises_ai_provider_error(): """All models in chain fail → AIProviderError raised.""" provider = _make_provider() provider._client.chat.completions.create.side_effect = RuntimeError("connection error") with pytest.raises(AIProviderError): provider._generate_json( task="explain", context="test", language="English", metadata=None, response_schema=SimpleExplanationAIOutput, route=("main",), ) def test_fatal_401_skips_retry(): """401 error → _FatalAPIError raised, no retry attempted.""" provider = _make_provider() auth_exc = Exception("Unauthorized") auth_exc.status_code = 401 # type: ignore[attr-defined] provider._client.chat.completions.create.side_effect = auth_exc with pytest.raises((AIProviderError, _FatalAPIError)): provider._generate_json( task="explain", context="test", language="English", metadata=None, response_schema=SimpleExplanationAIOutput, route=("main", "llama"), ) # Only one call made — retry skipped after fatal error assert provider._client.chat.completions.create.call_count == 1 def test_fallback_to_mock_on_generation_failure(): """When ai_fallback_to_mock=True, failed generation falls back to mock.""" provider = _make_provider() provider._settings = MagicMock() provider._settings.ai_fallback_to_mock = True provider._settings.ai_timeout_seconds = 30 provider._settings.openrouter_site_url = "https://docdoe.ai" provider._settings.openrouter_app_name = "DocDoe AI" provider._client.chat.completions.create.side_effect = RuntimeError("503 service unavailable") result = provider.generate_simple_explanation( context="photosynthesis is the process of making food", language="English", ) assert provider.is_fallback is True assert "simple_meaning" in result # mock provider always has this key def test_no_fallback_raises_on_failure(): """When ai_fallback_to_mock=False, failed generation raises AIProviderError.""" provider = _make_provider() provider._settings = MagicMock() provider._settings.ai_fallback_to_mock = False provider._settings.ai_timeout_seconds = 30 provider._settings.openrouter_site_url = "https://docdoe.ai" provider._settings.openrouter_app_name = "DocDoe AI" provider._client.chat.completions.create.side_effect = RuntimeError("model error") with pytest.raises(AIProviderError): provider.generate_simple_explanation( context="photosynthesis", language="English", ) assert provider.is_fallback is False def test_model_chain_notes_route(): """Notes route has at least 2 models (benchmark-corrected: gpt_oss/nemotron removed because they return HTTP 404 on the current OpenRouter account data-policy setting). Restore to 4 once openrouter.ai/settings/privacy is configured. """ provider = _make_provider() chain = provider._model_chain(provider._ROUTE_NOTES) assert len(chain) >= 2 assert len(set(chain)) == len(chain) # no duplicates def test_model_chain_flashcards_starts_with_main(): """Flashcards route starts with main model (benchmark-corrected: nemotron was previously primary but returns HTTP 404 — removed until data policy is set). """ provider = _make_provider() chain = provider._model_chain(provider._ROUTE_FLASHCARDS) assert chain[0] == provider._models["main"] def test_model_chain_pyq_starts_with_main(): """PYQ route starts with main model (benchmark-corrected: gpt_oss was previously primary but returns HTTP 404 — removed until data policy is set). """ provider = _make_provider() chain = provider._model_chain(provider._ROUTE_PYQ) assert chain[0] == provider._models["main"] def test_model_chain_no_empty_routes(): """Every route resolves to at least one model.""" provider = _make_provider() routes = [ provider._ROUTE_NOTES, provider._ROUTE_SIMPLE, provider._ROUTE_QUIZ, provider._ROUTE_FLASHCARDS, provider._ROUTE_EXAM, provider._ROUTE_PYQ, provider._ROUTE_VIDEO, provider._ROUTE_EXTRACT, ] for route in routes: chain = provider._model_chain(route) assert len(chain) >= 1, f"Route {route} resolved to empty chain"