tests/test_openrouter_json_reliability.py

"""
Tests for OpenRouter JSON parsing reliability and model-chain retry logic.

All tests run against mock provider by default.  Tests that exercise
OpenRouterAIProvider internals monkey-patch the HTTP client so no real
API calls are made.
"""

from __future__ import annotations

import json
import os
import sys
from pathlib import Path
from unittest.mock import MagicMock, patch

import pytest

BACKEND_DIR = Path(__file__).resolve().parents[1]
if str(BACKEND_DIR) not in sys.path:
    sys.path.insert(0, str(BACKEND_DIR))

os.environ.setdefault("AI_PROVIDER", "mock")
os.environ.setdefault("AUTH_ENABLED", "false")

from app.core.config import get_settings

get_settings.cache_clear()

from app.services.ai_provider import (
    AIProviderError,
    _FatalAPIError,
    _http_status_code,
    _parse_json_text,
    _validate_or_pass,
    MockAIProvider,
    OpenRouterAIProvider,
    SimpleExplanationAIOutput,
)


# ---------------------------------------------------------------------------
# _parse_json_text unit tests
# ---------------------------------------------------------------------------


def test_parse_plain_json():
    payload = {"simple_meaning": "test", "exam_answer": "ok"}
    assert _parse_json_text(json.dumps(payload)) == payload


def test_parse_markdown_fenced_json():
    payload = {"simple_meaning": "photosynthesis"}
    raw = f"```json\n{json.dumps(payload)}\n```"
    assert _parse_json_text(raw) == payload


def test_parse_fenced_json_no_language_tag():
    payload = {"title": "notes"}
    raw = f"```\n{json.dumps(payload)}\n```"
    assert _parse_json_text(raw) == payload


def test_parse_json_with_prose_prefix():
    """Model outputs extra text before the JSON object."""
    payload = {"simple_meaning": "hello"}
    raw = f"Sure! Here is the result:\n{json.dumps(payload)}\nHope that helps."
    result = _parse_json_text(raw)
    assert result == payload


def test_parse_json_with_prose_both_sides():
    payload = {"title": "EM Induction"}
    raw = f"Here you go:\n{json.dumps(payload)}\n\nLet me know if you need more."
    assert _parse_json_text(raw) == payload


def test_parse_invalid_json_raises():
    with pytest.raises(AIProviderError):
        _parse_json_text("This is not JSON at all")


def test_parse_json_array_raises():
    """JSON array is not an object — should raise."""
    with pytest.raises(AIProviderError):
        _parse_json_text(json.dumps([1, 2, 3]))


def test_parse_empty_string_raises():
    with pytest.raises(AIProviderError):
        _parse_json_text("")


# ---------------------------------------------------------------------------
# _validate_or_pass unit tests
# ---------------------------------------------------------------------------


def test_validate_good_data_returns_model_dump():
    data = {
        "simple_meaning": "Plants make food",
        "explain_like_15_year_old": "Like a kitchen for sunlight",
        "real_life_example": "Solar panels",
        "memory_trick": "SUN = SUNtain",
        "exam_answer": "Photosynthesis: green plants...",
    }
    result = _validate_or_pass(data, SimpleExplanationAIOutput)
    assert result["simple_meaning"] == "Plants make food"
    # Pydantic should fill in missing list fields with defaults
    assert isinstance(result["step_by_step"], list)


def test_validate_bad_data_returns_raw():
    """Validation failure should return raw dict, not crash."""
    raw = {"garbage_field": 999}
    result = _validate_or_pass(raw, SimpleExplanationAIOutput)
    # Should return something (either validated defaults or raw data)
    assert isinstance(result, dict)


# ---------------------------------------------------------------------------
# _http_status_code helper
# ---------------------------------------------------------------------------


def test_http_status_code_from_status_attribute():
    exc = Exception("bad")
    exc.status_code = 429  # type: ignore[attr-defined]
    assert _http_status_code(exc) == 429


def test_http_status_code_missing_returns_none():
    assert _http_status_code(ValueError("no code")) is None


# ---------------------------------------------------------------------------
# OpenRouterAIProvider model-chain retry logic (mocked)
# ---------------------------------------------------------------------------


def _make_provider(api_key: str = "test-key") -> OpenRouterAIProvider:
    """Create an OpenRouterAIProvider without calling __init__.

    Uses __new__ to bypass __init__ (which would try to import openai and
    create a real HTTP client).  All attributes are set manually so the
    provider is fully functional for unit tests that only touch logic.
    """
    os.environ["OPENROUTER_API_KEY"] = api_key
    get_settings.cache_clear()
    # __new__ skips __init__ — no real OpenAI client is created.
    provider = OpenRouterAIProvider.__new__(OpenRouterAIProvider)
    provider._settings = get_settings()
    provider._fallback = MockAIProvider()
    provider._models = {
        "main": "deepseek/deepseek-v4-flash:free",
        "llama": "meta-llama/llama-3.3-70b-instruct:free",
        "gpt_oss": "openai/gpt-oss-120b:free",
        "nemotron": "nvidia/nemotron-3-nano-30b-a3b:free",
    }
    provider._client = MagicMock()
    provider.model_name = "openrouter:deepseek/deepseek-v4-flash:free"
    provider.is_fallback = False
    provider.fallback_reason = None
    provider.last_error_code = None
    return provider


def _good_completion(content: str) -> MagicMock:
    choice = MagicMock()
    choice.message.content = content
    resp = MagicMock()
    resp.choices = [choice]
    return resp


def test_model_chain_retries_on_bad_json():
    """First model returns garbage JSON → second model succeeds."""
    provider = _make_provider()
    good_payload = json.dumps({
        "simple_meaning": "Plants make food using sunlight",
        "explain_like_15_year_old": "Like a food factory",
        "real_life_example": "Solar energy",
        "memory_trick": "SUN",
        "exam_answer": "Photosynthesis is...",
    })
    provider._client.chat.completions.create.side_effect = [
        _good_completion("NOT JSON AT ALL !!!"),  # first model fails json parse
        _good_completion(good_payload),           # second model succeeds
    ]
    result = provider._generate_json(
        task="explain",
        context="photosynthesis",
        language="English",
        metadata=None,
        response_schema=SimpleExplanationAIOutput,
        route=("main", "llama"),
    )
    assert "simple_meaning" in result
    assert provider.model_name.startswith("openrouter:")


def test_all_models_fail_raises_ai_provider_error():
    """All models in chain fail → AIProviderError raised."""
    provider = _make_provider()
    provider._client.chat.completions.create.side_effect = RuntimeError("connection error")
    with pytest.raises(AIProviderError):
        provider._generate_json(
            task="explain",
            context="test",
            language="English",
            metadata=None,
            response_schema=SimpleExplanationAIOutput,
            route=("main",),
        )


def test_fatal_401_skips_retry():
    """401 error → _FatalAPIError raised, no retry attempted."""
    provider = _make_provider()
    auth_exc = Exception("Unauthorized")
    auth_exc.status_code = 401  # type: ignore[attr-defined]
    provider._client.chat.completions.create.side_effect = auth_exc

    with pytest.raises((AIProviderError, _FatalAPIError)):
        provider._generate_json(
            task="explain",
            context="test",
            language="English",
            metadata=None,
            response_schema=SimpleExplanationAIOutput,
            route=("main", "llama"),
        )
    # Only one call made — retry skipped after fatal error
    assert provider._client.chat.completions.create.call_count == 1


def test_fallback_to_mock_on_generation_failure():
    """When ai_fallback_to_mock=True, failed generation falls back to mock."""
    provider = _make_provider()
    provider._settings = MagicMock()
    provider._settings.ai_fallback_to_mock = True
    provider._settings.ai_timeout_seconds = 30
    provider._settings.openrouter_site_url = "https://docdoe.ai"
    provider._settings.openrouter_app_name = "DocDoe AI"
    provider._client.chat.completions.create.side_effect = RuntimeError("503 service unavailable")

    result = provider.generate_simple_explanation(
        context="photosynthesis is the process of making food",
        language="English",
    )
    assert provider.is_fallback is True
    assert "simple_meaning" in result  # mock provider always has this key


def test_no_fallback_raises_on_failure():
    """When ai_fallback_to_mock=False, failed generation raises AIProviderError."""
    provider = _make_provider()
    provider._settings = MagicMock()
    provider._settings.ai_fallback_to_mock = False
    provider._settings.ai_timeout_seconds = 30
    provider._settings.openrouter_site_url = "https://docdoe.ai"
    provider._settings.openrouter_app_name = "DocDoe AI"
    provider._client.chat.completions.create.side_effect = RuntimeError("model error")

    with pytest.raises(AIProviderError):
        provider.generate_simple_explanation(
            context="photosynthesis",
            language="English",
        )
    assert provider.is_fallback is False


def test_model_chain_notes_route():
    """Notes route has at least 2 models (benchmark-corrected: gpt_oss/nemotron removed
    because they return HTTP 404 on the current OpenRouter account data-policy setting).
    Restore to 4 once openrouter.ai/settings/privacy is configured.
    """
    provider = _make_provider()
    chain = provider._model_chain(provider._ROUTE_NOTES)
    assert len(chain) >= 2
    assert len(set(chain)) == len(chain)  # no duplicates


def test_model_chain_flashcards_starts_with_main():
    """Flashcards route starts with main model (benchmark-corrected: nemotron
    was previously primary but returns HTTP 404 — removed until data policy is set).
    """
    provider = _make_provider()
    chain = provider._model_chain(provider._ROUTE_FLASHCARDS)
    assert chain[0] == provider._models["main"]


def test_model_chain_pyq_starts_with_main():
    """PYQ route starts with main model (benchmark-corrected: gpt_oss
    was previously primary but returns HTTP 404 — removed until data policy is set).
    """
    provider = _make_provider()
    chain = provider._model_chain(provider._ROUTE_PYQ)
    assert chain[0] == provider._models["main"]


def test_model_chain_no_empty_routes():
    """Every route resolves to at least one model."""
    provider = _make_provider()
    routes = [
        provider._ROUTE_NOTES,
        provider._ROUTE_SIMPLE,
        provider._ROUTE_QUIZ,
        provider._ROUTE_FLASHCARDS,
        provider._ROUTE_EXAM,
        provider._ROUTE_PYQ,
        provider._ROUTE_VIDEO,
        provider._ROUTE_EXTRACT,
    ]
    for route in routes:
        chain = provider._model_chain(route)
        assert len(chain) >= 1, f"Route {route} resolved to empty chain"