tests/test_output_quality_contract.py

"""Output Quality Contract tests for DocDoe AI prompts.

These tests pin the prompt/output policy promises made in the
'DOC DOE AI — GENERATED OUTPUT QUALITY SYSTEM PASS' spec (PART 12):

  1. material_only prompt forbids PYQ claims
  2. single_question_paper prompt forbids trend claims
  3. multiple_question_papers prompt allows only uploaded-paper comparison
  4. verified_10_year_pyq prompt allows 10-year language
  5. notes output prompt requests scoring keywords and common mistakes
  6. last-night plan prompt includes 30 min / 2 hour / half day
  7. quiz prompt includes answer + explanation + common trap
  8. flashcards include exam use
  9. study path does not invent profile values
 10. question paper prompt says one-paper limit when only one paper exists
"""
from __future__ import annotations

import inspect

import pytest

from app.services.ai_provider import (
    FlashcardAIItem,
    LastNightPlanAIOutput,
    NotesAIOutput,
    PreviousPaperAnalysisAIOutput,
    QuizAIQuestion,
    SYSTEM_INSTRUCTION,
    SarvamAIProvider,
    _build_prompt,
)
from app.services.evidence_contract import (
    MATERIAL_ONLY,
    MULTI_PAPER,
    SINGLE_PAPER,
    VERIFIED_10YR,
    _make_rules,
)


def _sarvam_task_source(method_name: str) -> str:
    """Return the source text of a SarvamAIProvider method (used to pin prompt task strings)."""
    method = getattr(SarvamAIProvider, method_name)
    return inspect.getsource(method)


# ── Evidence-policy prompt checks (spec rules 1-4) ──────────────────────────


class TestEvidencePromptPolicy:
    def test_material_only_forbids_pyq_pattern_phrasing(self) -> None:
        rules = _make_rules(MATERIAL_ONLY, 0, 0, None, None, None)
        assert "PROHIBITED" in rules
        assert "PYQ pattern" in rules
        assert "asked every year" in rules
        assert "Likely from this material" in rules

    def test_single_paper_forbids_trend_claims(self) -> None:
        rules = _make_rules(SINGLE_PAPER, 1, 1, None, None, None)
        assert "trend" in rules
        assert "In this uploaded paper" in rules
        # No "10-year" claim allowed at this level
        assert "PROHIBITED" in rules

    def test_multi_paper_allows_uploaded_compare_only(self) -> None:
        rules = _make_rules(MULTI_PAPER, 4, 4, None, None, None)
        assert "Across uploaded papers" in rules
        assert "10-year trend" in rules  # listed as PROHIBITED
        assert "PROHIBITED" in rules

    def test_verified_10yr_allows_10_year_language(self) -> None:
        rules = _make_rules(VERIFIED_10YR, 10, 10, None, None, None)
        assert "10-year PYQ pattern" in rules
        assert "PROHIBITED" not in rules  # verified set has no extra prohibitions


# ── Notes output contract (rule 5) ──────────────────────────────────────────


class TestNotesPromptContract:
    def test_notes_task_requires_scoring_keywords(self) -> None:
        src = _sarvam_task_source("generate_notes")
        assert "scoring_keywords" in src

    def test_notes_task_requires_common_mistakes(self) -> None:
        src = _sarvam_task_source("generate_notes")
        assert "common_mistakes" in src

    def test_notes_task_includes_next_study_action(self) -> None:
        src = _sarvam_task_source("generate_notes")
        assert "next_study_action" in src

    def test_notes_task_includes_ten_minute_revision(self) -> None:
        src = _sarvam_task_source("generate_notes")
        assert "ten_minute_revision" in src

    def test_notes_task_disallows_generic_dump(self) -> None:
        src = _sarvam_task_source("generate_notes")
        # Tell the model not to copy-paste / re-write the whole chapter
        assert "DO NOT rewrite the whole chapter" in src
        assert "DO NOT copy-paste textbook prose" in src

    def test_notes_schema_accepts_new_fields(self) -> None:
        instance = NotesAIOutput(
            title="X",
            student_level_summary="y",
            simple_explanation="z",
            scoring_keywords=["a", "b"],
            common_mistakes=["c"],
            ten_minute_revision=["d"],
            next_study_action="practice quiz",
            caution_note=None,
            source_summary="one-liner",
        )
        dumped = instance.model_dump()
        assert dumped["scoring_keywords"] == ["a", "b"]
        assert dumped["common_mistakes"] == ["c"]
        assert dumped["next_study_action"] == "practice quiz"


# ── Last-night plan contract (rule 6) ───────────────────────────────────────


class TestLastNightPromptContract:
    def test_last_night_includes_30_minute_block(self) -> None:
        src = _sarvam_task_source("generate_last_night_plan")
        assert "if_30_minutes" in src
        assert "30-minute" in src or "30 min" in src

    def test_last_night_includes_2_hour_block(self) -> None:
        src = _sarvam_task_source("generate_last_night_plan")
        assert "if_2_hours" in src
        assert "2-hour" in src or "2 hour" in src

    def test_last_night_includes_half_day_block(self) -> None:
        src = _sarvam_task_source("generate_last_night_plan")
        assert "if_half_day" in src

    def test_last_night_demands_do_first_and_skip(self) -> None:
        src = _sarvam_task_source("generate_last_night_plan")
        assert "do_first" in src
        assert "skip_or_skim" in src
        assert "quick_recall_questions" in src
        assert "final_check_before_sleep" in src

    def test_last_night_rejects_motivational_fluff(self) -> None:
        src = _sarvam_task_source("generate_last_night_plan")
        assert "NO motivational fluff" in src

    def test_last_night_schema_accepts_new_blocks(self) -> None:
        instance = LastNightPlanAIOutput(
            requested_minutes=120,
            total_minutes=120,
            calming_note="x",
            trust_note="y",
            if_30_minutes=["a"],
            if_2_hours=["b"],
            if_half_day=["c"],
            do_first=["d"],
            skip_or_skim=["e"],
            quick_recall_questions=["f?"],
            final_check_before_sleep=["g"],
            caution_note="Based on uploaded material only",
        )
        dumped = instance.model_dump()
        assert dumped["if_30_minutes"] == ["a"]
        assert dumped["if_2_hours"] == ["b"]
        assert dumped["if_half_day"] == ["c"]


# ── Quiz contract (rule 7) ──────────────────────────────────────────────────


class TestQuizPromptContract:
    def test_quiz_task_requires_common_trap(self) -> None:
        src = _sarvam_task_source("generate_quiz")
        assert "common_trap" in src

    def test_quiz_task_requires_why_this_matters(self) -> None:
        src = _sarvam_task_source("generate_quiz")
        assert "why_this_matters" in src

    def test_quiz_task_requires_explanation_and_answer(self) -> None:
        src = _sarvam_task_source("generate_quiz")
        assert "explanation" in src
        assert "answer" in src

    def test_quiz_task_forbids_fake_year_claims(self) -> None:
        src = _sarvam_task_source("generate_quiz")
        assert "asked every year" in src  # listed in negation
        assert "evidence policy" in src

    def test_quiz_schema_accepts_trap_and_why(self) -> None:
        q = QuizAIQuestion(
            question="x?",
            type="mcq",
            options=["A", "B", "C", "D"],
            answer="A",
            explanation="why",
            difficulty="easy",
            skill="recall",
            topic="t",
            why_this_matters="needed for 2-mark",
            common_trap="students pick B because it looks similar",
        )
        dumped = q.model_dump()
        assert dumped["common_trap"]
        assert dumped["why_this_matters"]


# ── Flashcards contract (rule 8) ────────────────────────────────────────────


class TestFlashcardsPromptContract:
    def test_flashcards_task_requires_exam_use(self) -> None:
        src = _sarvam_task_source("generate_flashcards")
        assert "exam_use" in src

    def test_flashcards_task_requires_memory_hook(self) -> None:
        src = _sarvam_task_source("generate_flashcards")
        assert "memory_hook" in src

    def test_flashcards_task_focuses_on_memory(self) -> None:
        src = _sarvam_task_source("generate_flashcards")
        assert "memory-focused" in src or "memory focused" in src
        assert "Avoid filler cards" in src

    def test_flashcards_schema_accepts_new_fields(self) -> None:
        card = FlashcardAIItem(
            front="x",
            back="y",
            hint="h",
            type="definition",
            difficulty="easy",
            topic="t",
            memory_hook="rhyme",
            exam_use="1-mark definition",
        )
        dumped = card.model_dump()
        assert dumped["memory_hook"] == "rhyme"
        assert dumped["exam_use"] == "1-mark definition"


# ── Question-paper analysis contract (rule 10) ──────────────────────────────


class TestQuestionPaperPromptContract:
    def test_analysis_prompt_routes_by_evidence_level(self) -> None:
        src = _sarvam_task_source("analyze_previous_papers")
        # Must branch by every evidence level
        assert "material_only" in src
        assert "single_question_paper" in src
        assert "multiple_question_papers" in src
        assert "verified_10_year_pyq" in src

    def test_analysis_prompt_says_one_paper_limit_for_single(self) -> None:
        src = _sarvam_task_source("analyze_previous_papers")
        assert "one_paper_limit_note" in src
        assert "Based on one uploaded paper only" in src
        assert "Upload more papers" in src

    def test_analysis_prompt_titles_per_evidence_level(self) -> None:
        src = _sarvam_task_source("analyze_previous_papers")
        assert "Likely questions from this material" in src
        assert "Signals from this paper" in src
        assert "Signals across uploaded papers" in src

    def test_analysis_prompt_forbids_invented_frequency(self) -> None:
        src = _sarvam_task_source("analyze_previous_papers")
        assert "Never invent frequencies, years, or boards" in src

    def test_analysis_schema_accepts_new_fields(self) -> None:
        out = PreviousPaperAnalysisAIOutput(
            evidence_label="Based on 1 uploaded question paper",
            one_paper_limit_note="Based on one uploaded paper only.",
            next_study_action="Practice 2-mark on chapter 4",
            caution_note=None,
        )
        dumped = out.model_dump()
        assert dumped["one_paper_limit_note"]
        assert dumped["next_study_action"]


# ── Study-path: no invented profile values (rule 9) ─────────────────────────


class TestStudyPathProfileSafety:
    """The study_path engine is rule-based; it MUST NOT fabricate board/grade."""

    def test_engine_outputs_only_provided_profile(self) -> None:
        from app.services.study_path_engine import build_study_path

        profile = {}  # empty profile — engine must not invent Kerala HSE
        analysis = {
            "subject": None,
            "topic": None,
            "goal": None,
            "time_left": None,
            "level": None,
            "language_preference": None,
        }
        result = build_study_path(
            study_profile=profile,
            analysis=analysis,
            source_context=None,
            pyq_summary={"available": False, "question_count": 0},
        )
        # Result text should never claim Kerala HSE or CBSE when not provided
        flat = repr(result).lower()
        assert "kerala hse" not in flat
        assert "cbse" not in flat


# ── _build_prompt header: surfaces new metadata fields ──────────────────────


class TestBuildPromptHeader:
    def test_includes_evidence_level_when_present(self) -> None:
        prompt = _build_prompt(
            task="Generate notes.",
            context="some context",
            language="English",
            metadata={
                "subject": "Biology",
                "evidence_level": "material_only",
                "evidence_label": "Based on uploaded material only",
                "material_type": "chapter",
                "source_title": "Genetics Chapter 5",
                "retrieval_confidence": 0.72,
                "board": "Kerala HSE",
                "class_level": "+2",
            },
        )
        assert "Evidence level: material_only" in prompt
        assert "Evidence label: Based on uploaded material only" in prompt
        assert "Material type: chapter" in prompt
        assert "Source title: Genetics Chapter 5" in prompt
        assert "Board: Kerala HSE" in prompt
        assert "Class/Grade: +2" in prompt
        assert "Retrieval confidence: 0.72" in prompt

    def test_does_not_hardcode_kerala_when_board_absent(self) -> None:
        prompt = _build_prompt(
            task="Generate notes.",
            context="some context",
            language="English",
            metadata={"subject": "Physics"},
        )
        # Header must not invent a board
        assert "Board: " not in prompt
        # Quality rules tell the model not to assume Kerala HSE
        assert "Do not assume Kerala HSE" in prompt

    def test_quality_rules_forbid_fake_pyq_claims(self) -> None:
        prompt = _build_prompt(
            task="Generate notes.",
            context="some context",
            language="English",
            metadata={},
        )
        assert "asked every year" in prompt
        assert "PYQ claims" in prompt


# ── SYSTEM_INSTRUCTION: neutral board posture ───────────────────────────────


class TestSystemInstruction:
    def test_system_instruction_is_board_neutral(self) -> None:
        # Must mention that board adapts to context, not default to Kerala HSE
        assert "DO NOT assume Kerala HSE" in SYSTEM_INSTRUCTION or "do NOT assume Kerala HSE" in SYSTEM_INSTRUCTION

    def test_system_instruction_forbids_as_an_ai(self) -> None:
        assert "'As an AI'" in SYSTEM_INSTRUCTION

    def test_system_instruction_forbids_fabricated_pyq(self) -> None:
        assert "PYQ frequencies" in SYSTEM_INSTRUCTION