tests/unit/test_faithfulness.py

"""Unit tests: Faithfulness checker — hallucination detection."""
import pytest
from unittest.mock import AsyncMock, patch, MagicMock


@pytest.mark.asyncio
async def test_faithful_response_passes():
    """A response directly supported by context should pass faithfulness check."""
    try:
        from app.core.rag.faithfulness import FaithfulnessChecker
        checker = FaithfulnessChecker()
        context = "The book costs $19.99 and is available on Amazon."
        response = "You can get the book on Amazon for $19.99."
        with patch.object(checker, "_score", new_callable=AsyncMock, return_value=0.92):
            result = await checker.check(response=response, context=context)
        assert result.passed is True
        assert result.score >= 0.55
    except (ImportError, AttributeError):
        pytest.skip("FaithfulnessChecker not importable with this interface")


@pytest.mark.asyncio
async def test_hallucinated_response_fails():
    """A response containing unsupported claims should fail faithfulness."""
    try:
        from app.core.rag.faithfulness import FaithfulnessChecker
        checker = FaithfulnessChecker()
        context = "The book is about a detective in London."
        response = "The book is a romance novel set in Paris and won the Nobel Prize."
        with patch.object(checker, "_score", new_callable=AsyncMock, return_value=0.12):
            result = await checker.check(response=response, context=context)
        assert result.passed is False
    except (ImportError, AttributeError):
        pytest.skip("FaithfulnessChecker not importable with this interface")


def test_faithfulness_threshold_from_config():
    """Threshold must match the config value — prevents silent config drift."""
    from app.config import get_settings
    cfg = get_settings()
    assert 0.0 < cfg.RAG_FAITHFULNESS_THRESHOLD < 1.0
    assert cfg.RAG_FAITHFULNESS_THRESHOLD == 0.55