"""Claude call sites that expect JSON ask for JSON mode, and every converted call carries an explicit timeout. json_output=True is what makes the helper demand a JSON-only reply and recover it from fences or prose, so a site that drops it starts failing on the first reply that opens with a sentence. The SDK's own timeout is 600 s, so a call without `timeout` can hold a worker thread (and the request behind it) for ten minutes. Each test replaces the helper a call site goes through and asserts on the request that reaches it. """ from __future__ import annotations import asyncio import json from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import pytest from rag import pdf_ingestion from routes import diagnostic, try_it_yourself from services import ( class_analytics_engine, intervention_engine, student_intelligence_pipeline, tutor_nudge_service, variance_engine, ) from services.ai_client import CHAT_MODEL PIPELINE_EVENT = student_intelligence_pipeline.StudentActivityEvent( student_id="student-1", event_type="quiz", occurred_at="2026-01-01T00:00:00Z" ) # Claude Haiku writes about this many output tokens per second; a call's token budget must be writable inside its timeout. HAIKU_OUTPUT_TOKENS_PER_SECOND = 250 # Quiz Battle rounds are clamped to 3-20, and the Cloud Function stops waiting for /api/quiz-battle/generate after # 30 s (functions/src/triggers/quizBattleApi.ts). BATTLE_MAX_ROUNDS = 20 BATTLE_CALLER_TIMEOUT_SECONDS = 30 # Each case: how to trigger the site, the reply the site can parse, and the timeout it must pass. # services.ai_client.complete_text is looked up at call time inside these modules, so patching it there reaches them. LAZY_IMPORT_SITES = [ pytest.param( lambda: intervention_engine.InterventionEngine()._generate_insights( grade_level="11", section="A", risk_level="high", avg_score=45.0, engagement="Low", strong_topics=[], weak_topics=["Factoring"], quiz_count=3, ), '{"learning_strengths": "Persistent.", "next_steps_summary": "Practice factoring."}', id="intervention-insights", ), pytest.param( lambda: intervention_engine.InterventionEngine()._generate_learning_path( student_id="student-1", weak_topics=["Factoring"], weakest_topic="Factoring", engagement="Low", risk_level="high", grade_level="11", topic_avgs={"Factoring": 40}, ), '{"steps": [], "ai_rationale": "Start with factoring."}', id="intervention-learning-path", ), pytest.param( lambda: intervention_engine.InterventionEngine()._generate_recommendations( grade_level="11", risk_level="high", weak_topics=["Factoring"], avg_score=45.0 ), '["Review factoring in a one-on-one session."]', id="intervention-recommendations", ), pytest.param( lambda: class_analytics_engine.ClassAnalyticsEngine()._generate_insights( class_id="class-1", class_name="Newton", grade_level="11", section="A", student_count=30, class_average=70.0, completion_rate=80.0, participation_rate=90.0, risk_dist={}, topic_perf=[], ), '{"class_summary": "Steady progress."}', id="class-analytics", ), pytest.param( lambda: student_intelligence_pipeline.StudentIntelligencePipeline()._generate_ai_context({}, PIPELINE_EVENT), '{"ai_summary": "On track."}', id="student-intelligence-pipeline", ), ] @pytest.mark.parametrize(("trigger", "reply"), LAZY_IMPORT_SITES) def test_engine_sites_request_json_output_with_a_60_second_timeout(trigger, reply) -> None: with patch("services.ai_client.complete_text", return_value=reply) as complete_text: asyncio.run(trigger()) sent = complete_text.call_args.kwargs assert sent["json_output"] is True assert sent["timeout"] == 60.0 assert sent["model"] == CHAT_MODEL def test_intervention_learning_path_leaves_room_for_a_six_step_plan() -> None: reply = '{"steps": []}' with patch("services.ai_client.complete_text", return_value=reply) as complete_text: asyncio.run( intervention_engine.InterventionEngine()._generate_learning_path( student_id="student-1", weak_topics=["Factoring"], weakest_topic="Factoring", engagement="Low", risk_level="high", grade_level="11", topic_avgs={"Factoring": 40}, ) ) assert complete_text.call_args.kwargs["max_tokens"] == 1500 def test_diagnostic_questions_request_json_output_with_a_90_second_timeout() -> None: with patch.object(diagnostic, "complete_text", return_value="[]") as complete_text: assert asyncio.run(diagnostic._call_claude("system prompt", "user message")) == "[]" sent = complete_text.call_args.kwargs assert sent["json_output"] is True assert sent["timeout"] == 90.0 def test_variance_engine_requests_json_output_with_a_120_second_timeout() -> None: question = { "question": "What is 2+2?", "choices": ["A) 3", "B) 4", "C) 5", "D) 6"], "correct_answer": "B", "topic": "math", "grade_level": 8, } varied = [{**question, "variance_applied": ["paraphrase"]}] with ( patch.object(variance_engine, "get_cached_session", new=AsyncMock(return_value=None)), patch.object(variance_engine, "cache_session_questions", new=AsyncMock()), patch.object(variance_engine, "complete_text", return_value=json.dumps(varied)) as complete_text, ): asyncio.run(variance_engine.apply_variance([question], "session-1")) sent = complete_text.call_args.kwargs assert sent["json_output"] is True assert sent["timeout"] == 120.0 def test_variance_engine_fits_a_full_battle_inside_the_callers_wait() -> None: """A rewrite repeats every question in full (~300 tokens each): a 20-round battle does not fit the old 4000-token budget, and a budget the model cannot write within the Cloud Function's wait would lose the whole rewrite.""" question = { "question": "What is 2+2?", "choices": ["A) 3", "B) 4", "C) 5", "D) 6"], "correct_answer": "B", "topic": "math", "grade_level": 8, } battle = [{**question, "question": f"What is {number}+2?"} for number in range(BATTLE_MAX_ROUNDS)] varied = [{**original, "variance_applied": ["paraphrase"]} for original in battle] with ( patch.object(variance_engine, "get_cached_session", new=AsyncMock(return_value=None)), patch.object(variance_engine, "cache_session_questions", new=AsyncMock()), patch.object(variance_engine, "complete_text", return_value=json.dumps(varied)) as complete_text, ): asyncio.run(variance_engine.apply_variance(battle, "session-1")) sent = complete_text.call_args.kwargs assert BATTLE_MAX_ROUNDS * 300 <= sent["max_tokens"] assert sent["max_tokens"] / HAIKU_OUTPUT_TOKENS_PER_SECOND < BATTLE_CALLER_TIMEOUT_SECONDS def test_chunk_question_generation_requests_json_output_with_a_120_second_timeout() -> None: with patch.object(pdf_ingestion, "complete_text", return_value="[]") as complete_text: asyncio.run(pdf_ingestion._generate_questions_for_chunk("chunk text", "chunk-1", "linear_equations", 8)) sent = complete_text.call_args.kwargs assert sent["json_output"] is True assert sent["timeout"] == 120.0 def test_tutor_nudge_call_has_a_60_second_timeout() -> None: firestore = MagicMock() with ( patch.object(tutor_nudge_service, "_get_db", return_value=firestore), patch.object(tutor_nudge_service, "_has_recent_nudge", return_value=False), patch("services.ai_client.complete_text", return_value="Try five factoring problems today.") as complete_text, ): nudge = asyncio.run(tutor_nudge_service.generate_tutor_nudge_for_student("student-1", ["Factoring"])) assert nudge["message"] == "Try five factoring problems today." assert complete_text.call_args.kwargs["timeout"] == 60.0 def test_shadow_retry_leaves_room_for_five_variants() -> None: """Five questions with three hints each do not fit the old 2000-token budget.""" seen_requests = [] class RecordingClient: def generate_from_messages(self, inference_request: object) -> str: seen_requests.append(inference_request) return "[]" request = SimpleNamespace(state=SimpleNamespace(user=SimpleNamespace(uid="student-1"))) body = try_it_yourself.ShadowRetryRequest( userId="student-1", sessionId="session-1", struggleTopics=["Factoring"], subject="General Mathematics", count=5 ) with ( patch("routes.quiz_generation_routes._get_inference_client", return_value=RecordingClient()), patch("routes.quiz_generation_routes._parse_quiz_response", return_value=[]), patch("rag.curriculum_rag.retrieve_curriculum_context", return_value=[]), ): asyncio.run(try_it_yourself.generate_shadow_retries(request, body)) (inference_request,) = seen_requests assert inference_request.max_new_tokens == 4096