"""Output Quality Contract tests for DocDoe AI prompts. These tests pin the prompt/output policy promises made in the 'DOC DOE AI — GENERATED OUTPUT QUALITY SYSTEM PASS' spec (PART 12): 1. material_only prompt forbids PYQ claims 2. single_question_paper prompt forbids trend claims 3. multiple_question_papers prompt allows only uploaded-paper comparison 4. verified_10_year_pyq prompt allows 10-year language 5. notes output prompt requests scoring keywords and common mistakes 6. last-night plan prompt includes 30 min / 2 hour / half day 7. quiz prompt includes answer + explanation + common trap 8. flashcards include exam use 9. study path does not invent profile values 10. question paper prompt says one-paper limit when only one paper exists """ from __future__ import annotations import inspect import pytest from app.services.ai_provider import ( FlashcardAIItem, LastNightPlanAIOutput, NotesAIOutput, PreviousPaperAnalysisAIOutput, QuizAIQuestion, SYSTEM_INSTRUCTION, SarvamAIProvider, _build_prompt, ) from app.services.evidence_contract import ( MATERIAL_ONLY, MULTI_PAPER, SINGLE_PAPER, VERIFIED_10YR, _make_rules, ) def _sarvam_task_source(method_name: str) -> str: """Return the source text of a SarvamAIProvider method (used to pin prompt task strings).""" method = getattr(SarvamAIProvider, method_name) return inspect.getsource(method) # ── Evidence-policy prompt checks (spec rules 1-4) ────────────────────────── class TestEvidencePromptPolicy: def test_material_only_forbids_pyq_pattern_phrasing(self) -> None: rules = _make_rules(MATERIAL_ONLY, 0, 0, None, None, None) assert "PROHIBITED" in rules assert "PYQ pattern" in rules assert "asked every year" in rules assert "Likely from this material" in rules def test_single_paper_forbids_trend_claims(self) -> None: rules = _make_rules(SINGLE_PAPER, 1, 1, None, None, None) assert "trend" in rules assert "In this uploaded paper" in rules # No "10-year" claim allowed at this level assert "PROHIBITED" in rules def test_multi_paper_allows_uploaded_compare_only(self) -> None: rules = _make_rules(MULTI_PAPER, 4, 4, None, None, None) assert "Across uploaded papers" in rules assert "10-year trend" in rules # listed as PROHIBITED assert "PROHIBITED" in rules def test_verified_10yr_allows_10_year_language(self) -> None: rules = _make_rules(VERIFIED_10YR, 10, 10, None, None, None) assert "10-year PYQ pattern" in rules assert "PROHIBITED" not in rules # verified set has no extra prohibitions # ── Notes output contract (rule 5) ────────────────────────────────────────── class TestNotesPromptContract: def test_notes_task_requires_scoring_keywords(self) -> None: src = _sarvam_task_source("generate_notes") assert "scoring_keywords" in src def test_notes_task_requires_common_mistakes(self) -> None: src = _sarvam_task_source("generate_notes") assert "common_mistakes" in src def test_notes_task_includes_next_study_action(self) -> None: src = _sarvam_task_source("generate_notes") assert "next_study_action" in src def test_notes_task_includes_ten_minute_revision(self) -> None: src = _sarvam_task_source("generate_notes") assert "ten_minute_revision" in src def test_notes_task_disallows_generic_dump(self) -> None: src = _sarvam_task_source("generate_notes") # Tell the model not to copy-paste / re-write the whole chapter assert "DO NOT rewrite the whole chapter" in src assert "DO NOT copy-paste textbook prose" in src def test_notes_schema_accepts_new_fields(self) -> None: instance = NotesAIOutput( title="X", student_level_summary="y", simple_explanation="z", scoring_keywords=["a", "b"], common_mistakes=["c"], ten_minute_revision=["d"], next_study_action="practice quiz", caution_note=None, source_summary="one-liner", ) dumped = instance.model_dump() assert dumped["scoring_keywords"] == ["a", "b"] assert dumped["common_mistakes"] == ["c"] assert dumped["next_study_action"] == "practice quiz" # ── Last-night plan contract (rule 6) ─────────────────────────────────────── class TestLastNightPromptContract: def test_last_night_includes_30_minute_block(self) -> None: src = _sarvam_task_source("generate_last_night_plan") assert "if_30_minutes" in src assert "30-minute" in src or "30 min" in src def test_last_night_includes_2_hour_block(self) -> None: src = _sarvam_task_source("generate_last_night_plan") assert "if_2_hours" in src assert "2-hour" in src or "2 hour" in src def test_last_night_includes_half_day_block(self) -> None: src = _sarvam_task_source("generate_last_night_plan") assert "if_half_day" in src def test_last_night_demands_do_first_and_skip(self) -> None: src = _sarvam_task_source("generate_last_night_plan") assert "do_first" in src assert "skip_or_skim" in src assert "quick_recall_questions" in src assert "final_check_before_sleep" in src def test_last_night_rejects_motivational_fluff(self) -> None: src = _sarvam_task_source("generate_last_night_plan") assert "NO motivational fluff" in src def test_last_night_schema_accepts_new_blocks(self) -> None: instance = LastNightPlanAIOutput( requested_minutes=120, total_minutes=120, calming_note="x", trust_note="y", if_30_minutes=["a"], if_2_hours=["b"], if_half_day=["c"], do_first=["d"], skip_or_skim=["e"], quick_recall_questions=["f?"], final_check_before_sleep=["g"], caution_note="Based on uploaded material only", ) dumped = instance.model_dump() assert dumped["if_30_minutes"] == ["a"] assert dumped["if_2_hours"] == ["b"] assert dumped["if_half_day"] == ["c"] # ── Quiz contract (rule 7) ────────────────────────────────────────────────── class TestQuizPromptContract: def test_quiz_task_requires_common_trap(self) -> None: src = _sarvam_task_source("generate_quiz") assert "common_trap" in src def test_quiz_task_requires_why_this_matters(self) -> None: src = _sarvam_task_source("generate_quiz") assert "why_this_matters" in src def test_quiz_task_requires_explanation_and_answer(self) -> None: src = _sarvam_task_source("generate_quiz") assert "explanation" in src assert "answer" in src def test_quiz_task_forbids_fake_year_claims(self) -> None: src = _sarvam_task_source("generate_quiz") assert "asked every year" in src # listed in negation assert "evidence policy" in src def test_quiz_schema_accepts_trap_and_why(self) -> None: q = QuizAIQuestion( question="x?", type="mcq", options=["A", "B", "C", "D"], answer="A", explanation="why", difficulty="easy", skill="recall", topic="t", why_this_matters="needed for 2-mark", common_trap="students pick B because it looks similar", ) dumped = q.model_dump() assert dumped["common_trap"] assert dumped["why_this_matters"] # ── Flashcards contract (rule 8) ──────────────────────────────────────────── class TestFlashcardsPromptContract: def test_flashcards_task_requires_exam_use(self) -> None: src = _sarvam_task_source("generate_flashcards") assert "exam_use" in src def test_flashcards_task_requires_memory_hook(self) -> None: src = _sarvam_task_source("generate_flashcards") assert "memory_hook" in src def test_flashcards_task_focuses_on_memory(self) -> None: src = _sarvam_task_source("generate_flashcards") assert "memory-focused" in src or "memory focused" in src assert "Avoid filler cards" in src def test_flashcards_schema_accepts_new_fields(self) -> None: card = FlashcardAIItem( front="x", back="y", hint="h", type="definition", difficulty="easy", topic="t", memory_hook="rhyme", exam_use="1-mark definition", ) dumped = card.model_dump() assert dumped["memory_hook"] == "rhyme" assert dumped["exam_use"] == "1-mark definition" # ── Question-paper analysis contract (rule 10) ────────────────────────────── class TestQuestionPaperPromptContract: def test_analysis_prompt_routes_by_evidence_level(self) -> None: src = _sarvam_task_source("analyze_previous_papers") # Must branch by every evidence level assert "material_only" in src assert "single_question_paper" in src assert "multiple_question_papers" in src assert "verified_10_year_pyq" in src def test_analysis_prompt_says_one_paper_limit_for_single(self) -> None: src = _sarvam_task_source("analyze_previous_papers") assert "one_paper_limit_note" in src assert "Based on one uploaded paper only" in src assert "Upload more papers" in src def test_analysis_prompt_titles_per_evidence_level(self) -> None: src = _sarvam_task_source("analyze_previous_papers") assert "Likely questions from this material" in src assert "Signals from this paper" in src assert "Signals across uploaded papers" in src def test_analysis_prompt_forbids_invented_frequency(self) -> None: src = _sarvam_task_source("analyze_previous_papers") assert "Never invent frequencies, years, or boards" in src def test_analysis_schema_accepts_new_fields(self) -> None: out = PreviousPaperAnalysisAIOutput( evidence_label="Based on 1 uploaded question paper", one_paper_limit_note="Based on one uploaded paper only.", next_study_action="Practice 2-mark on chapter 4", caution_note=None, ) dumped = out.model_dump() assert dumped["one_paper_limit_note"] assert dumped["next_study_action"] # ── Study-path: no invented profile values (rule 9) ───────────────────────── class TestStudyPathProfileSafety: """The study_path engine is rule-based; it MUST NOT fabricate board/grade.""" def test_engine_outputs_only_provided_profile(self) -> None: from app.services.study_path_engine import build_study_path profile = {} # empty profile — engine must not invent Kerala HSE analysis = { "subject": None, "topic": None, "goal": None, "time_left": None, "level": None, "language_preference": None, } result = build_study_path( study_profile=profile, analysis=analysis, source_context=None, pyq_summary={"available": False, "question_count": 0}, ) # Result text should never claim Kerala HSE or CBSE when not provided flat = repr(result).lower() assert "kerala hse" not in flat assert "cbse" not in flat # ── _build_prompt header: surfaces new metadata fields ────────────────────── class TestBuildPromptHeader: def test_includes_evidence_level_when_present(self) -> None: prompt = _build_prompt( task="Generate notes.", context="some context", language="English", metadata={ "subject": "Biology", "evidence_level": "material_only", "evidence_label": "Based on uploaded material only", "material_type": "chapter", "source_title": "Genetics Chapter 5", "retrieval_confidence": 0.72, "board": "Kerala HSE", "class_level": "+2", }, ) assert "Evidence level: material_only" in prompt assert "Evidence label: Based on uploaded material only" in prompt assert "Material type: chapter" in prompt assert "Source title: Genetics Chapter 5" in prompt assert "Board: Kerala HSE" in prompt assert "Class/Grade: +2" in prompt assert "Retrieval confidence: 0.72" in prompt def test_does_not_hardcode_kerala_when_board_absent(self) -> None: prompt = _build_prompt( task="Generate notes.", context="some context", language="English", metadata={"subject": "Physics"}, ) # Header must not invent a board assert "Board: " not in prompt # Quality rules tell the model not to assume Kerala HSE assert "Do not assume Kerala HSE" in prompt def test_quality_rules_forbid_fake_pyq_claims(self) -> None: prompt = _build_prompt( task="Generate notes.", context="some context", language="English", metadata={}, ) assert "asked every year" in prompt assert "PYQ claims" in prompt # ── SYSTEM_INSTRUCTION: neutral board posture ─────────────────────────────── class TestSystemInstruction: def test_system_instruction_is_board_neutral(self) -> None: # Must mention that board adapts to context, not default to Kerala HSE assert "DO NOT assume Kerala HSE" in SYSTEM_INSTRUCTION or "do NOT assume Kerala HSE" in SYSTEM_INSTRUCTION def test_system_instruction_forbids_as_an_ai(self) -> None: assert "'As an AI'" in SYSTEM_INSTRUCTION def test_system_instruction_forbids_fabricated_pyq(self) -> None: assert "PYQ frequencies" in SYSTEM_INSTRUCTION