tests/test_cross_validation_numeric_runner.py

import asyncio
from app.schemas.cross_validation import AnswerKind, TaskType, EngineStatus, EquivalenceStatus, ConfidenceImpact, CrossValidationReport, LlmSolvedAnswer, ValidationConfidence
from app.services.cross_validation_numeric_runner import run_solution_set_cross_validation_numeric_check


class FakeContext:
    def model_dump(self, mode="json"):
        return {"statement": "Résoudre 3x - 9 = 0.", "max_score": 2}


async def fake_translate(**kwargs):
    class Candidate:
        wolfram_query = "Solve[3 x - 9 == 0, x]"
    return Candidate()


async def fake_solve(**kwargs):
    return LlmSolvedAnswer(status=EngineStatus.OK, answer_kind=AnswerKind.SOLUTION_SET, solved_answer="{3}", reasoning_summary="ok", confidence=ValidationConfidence.CERTAIN, needs_human_review=False, reason="ok")


async def fake_cross(**kwargs):
    return CrossValidationReport(student_extracted_answer="3", llm_solved_answer="{3}", llm_solution_status=EngineStatus.OK, wolfram_query="Solve[3 x - 9 == 0, x]", wolfram_status=EngineStatus.OK, wolfram_reference_result="x = 3", student_vs_wolfram=EquivalenceStatus.EQUIVALENT, llm_vs_wolfram=EquivalenceStatus.EQUIVALENT, student_vs_llm=EquivalenceStatus.EQUIVALENT, comparison_method="controlled_school_rule", confidence_impact=ConfidenceImpact.INCREASE, needs_human_review=False, reason="consistent")


def test_runner_returns_valid_numeric_check(monkeypatch):
    import app.services.cross_validation_numeric_runner as runner
    monkeypatch.setattr(runner, "translate_exercise_to_wolfram_with_llm", fake_translate)
    monkeypatch.setattr(runner, "solve_exercise_independently_with_llm", fake_solve)
    monkeypatch.setattr(runner, "run_cross_validation_with_wolfram_reference", fake_cross)
    check = asyncio.run(run_solution_set_cross_validation_numeric_check(answer_text="La reponse est {3}.", pedagogical_context=FakeContext()))
    assert check.is_valid is True
    assert check.check_type == "equivalence"
    assert check.confidence_impact == "increase"


async def fake_translate_derivative(**kwargs):
    class Candidate:
        task_type = TaskType.DERIVATIVE
        expected_answer_kind = AnswerKind.VALUE
        wolfram_query = "D[x^2, x]"
    return Candidate()


async def fail_if_called(**kwargs):
    raise AssertionError("solver_or_cross_validator_should_not_be_called")


def test_runner_skips_incompatible_derivative_task(monkeypatch):
    import app.services.cross_validation_numeric_runner as runner
    monkeypatch.setattr(runner, "translate_exercise_to_wolfram_with_llm", fake_translate_derivative)
    monkeypatch.setattr(runner, "solve_exercise_independently_with_llm", fail_if_called)
    monkeypatch.setattr(runner, "run_cross_validation_with_wolfram_reference", fail_if_called)
    check = asyncio.run(run_solution_set_cross_validation_numeric_check(answer_text="La reponse est {3}.", pedagogical_context=FakeContext()))
    assert check.is_valid is True
    assert check.wolfram_status == "skipped"
    assert check.cas_status == "NO_CAS_VALIDATION"
    assert check.confidence_impact == "neutral"
    assert check.human_review_reason is None
    assert "cross_validation_skipped_incompatible_task=derivative" in check.details
    assert "answer_kind=value" in check.details