Spaces:
Runtime error
Runtime error
File size: 3,252 Bytes
2cb9994 a64af0c 2cb9994 a64af0c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | import asyncio
from app.schemas.cross_validation import AnswerKind, TaskType, EngineStatus, EquivalenceStatus, ConfidenceImpact, CrossValidationReport, LlmSolvedAnswer, ValidationConfidence
from app.services.cross_validation_numeric_runner import run_solution_set_cross_validation_numeric_check
class FakeContext:
def model_dump(self, mode="json"):
return {"statement": "Résoudre 3x - 9 = 0.", "max_score": 2}
async def fake_translate(**kwargs):
class Candidate:
wolfram_query = "Solve[3 x - 9 == 0, x]"
return Candidate()
async def fake_solve(**kwargs):
return LlmSolvedAnswer(status=EngineStatus.OK, answer_kind=AnswerKind.SOLUTION_SET, solved_answer="{3}", reasoning_summary="ok", confidence=ValidationConfidence.CERTAIN, needs_human_review=False, reason="ok")
async def fake_cross(**kwargs):
return CrossValidationReport(student_extracted_answer="3", llm_solved_answer="{3}", llm_solution_status=EngineStatus.OK, wolfram_query="Solve[3 x - 9 == 0, x]", wolfram_status=EngineStatus.OK, wolfram_reference_result="x = 3", student_vs_wolfram=EquivalenceStatus.EQUIVALENT, llm_vs_wolfram=EquivalenceStatus.EQUIVALENT, student_vs_llm=EquivalenceStatus.EQUIVALENT, comparison_method="controlled_school_rule", confidence_impact=ConfidenceImpact.INCREASE, needs_human_review=False, reason="consistent")
def test_runner_returns_valid_numeric_check(monkeypatch):
import app.services.cross_validation_numeric_runner as runner
monkeypatch.setattr(runner, "translate_exercise_to_wolfram_with_llm", fake_translate)
monkeypatch.setattr(runner, "solve_exercise_independently_with_llm", fake_solve)
monkeypatch.setattr(runner, "run_cross_validation_with_wolfram_reference", fake_cross)
check = asyncio.run(run_solution_set_cross_validation_numeric_check(answer_text="La reponse est {3}.", pedagogical_context=FakeContext()))
assert check.is_valid is True
assert check.check_type == "equivalence"
assert check.confidence_impact == "increase"
async def fake_translate_derivative(**kwargs):
class Candidate:
task_type = TaskType.DERIVATIVE
expected_answer_kind = AnswerKind.VALUE
wolfram_query = "D[x^2, x]"
return Candidate()
async def fail_if_called(**kwargs):
raise AssertionError("solver_or_cross_validator_should_not_be_called")
def test_runner_skips_incompatible_derivative_task(monkeypatch):
import app.services.cross_validation_numeric_runner as runner
monkeypatch.setattr(runner, "translate_exercise_to_wolfram_with_llm", fake_translate_derivative)
monkeypatch.setattr(runner, "solve_exercise_independently_with_llm", fail_if_called)
monkeypatch.setattr(runner, "run_cross_validation_with_wolfram_reference", fail_if_called)
check = asyncio.run(run_solution_set_cross_validation_numeric_check(answer_text="La reponse est {3}.", pedagogical_context=FakeContext()))
assert check.is_valid is True
assert check.wolfram_status == "skipped"
assert check.cas_status == "NO_CAS_VALIDATION"
assert check.confidence_impact == "neutral"
assert check.human_review_reason is None
assert "cross_validation_skipped_incompatible_task=derivative" in check.details
assert "answer_kind=value" in check.details
|