tests/test_architecture_contest.py

"""Tests for contesting an assumption - step 3 of the defensible recommender.

Focused on the parts that do not need a live model: that the verdict stays
consistent with the boolean it summarises, and that a nullable prose field the
model answered with the word "None" does not reach the user as text.
"""
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from app.core.schemas import ArchitectureOption
from app.services.architecture_contest import ContestOutcome, contest_assumption


def _option() -> ArchitectureOption:
    return ArchitectureOption(
        id="opt_1",
        name="Modular Monolith",
        description="A single deployable service with internal module boundaries.",
        components=["api", "worker"],
        tech_stack={"backend": "Node.js"},
        pros=["one deploy target"],
        cons=["scales vertically first"],
        assumptions=["traffic is evenly distributed"],
    )


def _outcome(**overrides) -> ContestOutcome:
    payload = {
        "assumption_was_wrong": True,
        "changes_recommendation": False,
        "impact": (
            "The correction is right about the concurrency figure, but a modular "
            "monolith still handles this load without re-architecture."
        ),
        "affected_dimensions": ["scalability"],
        "revised_assumption": None,
        "still_recommended": True,
        "follow_up_question": None,
    }
    payload.update(overrides)
    return ContestOutcome(**payload)


async def _contest(outcome: ContestOutcome) -> dict:
    """Run contest_assumption with the model's reply stubbed out.

    `with_structured_output` is called synchronously by the service, so the LLM
    double must be a MagicMock - an AsyncMock returns a coroutine there, the
    service raises, and every assertion then reads the error fallback instead of
    the outcome under test.
    """
    structured = MagicMock()
    structured.ainvoke = AsyncMock(return_value=outcome)
    llm = MagicMock()
    llm.with_structured_output.return_value = structured

    with patch("app.services.architecture_contest.get_chat_model", return_value=llm):
        result = await contest_assumption(
            option=_option(),
            assumption="traffic is evenly distributed",
            correction="all 5000 users arrive in the same 20 minutes",
            requirements="5000 users year one.",
        )

    # The fallback also reports verdict "defended". Without this guard a broken
    # double would let these tests pass while exercising nothing.
    assert "error" not in result, f"hit the error fallback: {result.get('impact')}"
    return result


class TestVerdictIsDerived:
    @pytest.mark.asyncio
    async def test_holding_the_conclusion_reads_as_defended(self):
        result = await _contest(_outcome(changes_recommendation=False))
        assert result["verdict"] == "defended"

    @pytest.mark.asyncio
    async def test_changing_the_conclusion_reads_as_revised(self):
        result = await _contest(_outcome(changes_recommendation=True))
        assert result["verdict"] == "revised"

    @pytest.mark.asyncio
    async def test_accepting_a_fact_does_not_imply_revising(self):
        """The distinction the split schema exists to preserve."""
        result = await _contest(
            _outcome(assumption_was_wrong=True, changes_recommendation=False)
        )
        assert result["assumption_was_wrong"] is True
        assert result["verdict"] == "defended"
        assert result["still_recommended"] is True


class TestPlaceholderProseIsNormalized:
    @pytest.mark.asyncio
    @pytest.mark.parametrize("written", ["None", "none", "N/A", "null", "-"])
    async def test_revised_assumption_placeholder_becomes_null(self, written):
        """Rendered unguarded, this showed the user 'Now assumes: None'."""
        result = await _contest(_outcome(revised_assumption=written))
        assert result["revised_assumption"] is None

    @pytest.mark.asyncio
    @pytest.mark.parametrize("written", ["None", "n/a", "not applicable"])
    async def test_follow_up_question_placeholder_becomes_null(self, written):
        result = await _contest(_outcome(follow_up_question=written))
        assert result["follow_up_question"] is None

    @pytest.mark.asyncio
    async def test_real_prose_is_preserved(self):
        result = await _contest(
            _outcome(
                revised_assumption="Traffic spikes to 2000+ concurrent on Saturdays.",
                follow_up_question="What is the p99 latency budget for search?",
            )
        )
        assert result["revised_assumption"] == "Traffic spikes to 2000+ concurrent on Saturdays."
        assert result["follow_up_question"] == "What is the p99 latency budget for search?"


class TestFailureIsSurfaced:
    @pytest.mark.asyncio
    async def test_a_failed_reassessment_does_not_invent_agreement(self):
        """Falling back must not look like the model considered and agreed."""
        llm = MagicMock()
        llm.with_structured_output.side_effect = RuntimeError("provider down")

        with patch("app.services.architecture_contest.get_chat_model", return_value=llm):
            result = await contest_assumption(
                option=_option(),
                assumption="traffic is evenly distributed",
                correction="it is not",
                requirements="5000 users year one.",
            )

        assert result["error"] is True
        assert result["changes_recommendation"] is False
        assert "unavailable" in result["impact"].lower()