🚀 Auto-deploy backend from GitHub (4fdbab1)

rag/firebase_storage_loader.py

@@ -5,6 +5,7 @@ Downloads PDFs from Firebase Storage and extracts text for ChromaDB indexing.
 
 from __future__ import annotations
 
+import datetime
 import logging
 import os
 from pathlib import Path
@@ -148,4 +149,72 @@ PDF_METADATA: Dict[str, dict] = {
         "quarter": 1,
         "storage_path": "curriculum/stat_prob/SDO_Navotas_STAT_PROB_SHS_1stSem.FV.pdf",
     },
-}
+}
+
+
+def generate_signed_download_url(storage_path: str, expiration_hours: int = 24) -> Optional[str]:
+    """Generate a signed download URL for a Firebase Storage blob.
+
+    Args:
+        storage_path: The path of the blob in Firebase Storage.
+        expiration_hours: Number of hours until the URL expires (default 24).
+
+    Returns:
+        Signed URL string, or None if Firebase Storage is unavailable.
+    """
+    _, bucket = _init_firebase_storage()
+    if bucket is None:
+        logger.warning("Firebase Storage not available, cannot generate signed URL")
+        return None
+
+    try:
+        blob = bucket.blob(storage_path)
+        signed_url = blob.generate_signed_url(
+            expiration=datetime.timedelta(hours=expiration_hours),
+            method="GET",
+        )
+        logger.info("Generated signed URL for %s (expires in %dh)", storage_path, expiration_hours)
+        return signed_url
+    except Exception as e:
+        logger.error("Failed to generate signed URL for %s: %s", storage_path, e)
+        return None
+
+
+def get_study_materials_from_chunks(chunks: list[dict]) -> list[dict]:
+    """Extract study materials from chunks, deduplicating by source PDF.
+
+    Args:
+        chunks: List of chunk dicts with optional `source_file`, `storage_path`,
+               and `content_domain` keys.
+
+    Returns:
+        List of dicts with keys: `title`, `source_pdf_url`, `topic_match`.
+    """
+    seen_sources: set[str] = set()
+    materials: list[dict] = []
+
+    for chunk in chunks:
+        source = chunk.get("source_file")
+        if not source or source in seen_sources:
+            continue
+        seen_sources.add(source)
+
+        # Look up PDF metadata by storage_path
+        metadata = PDF_METADATA.get(source)
+        if metadata:
+            title = metadata.get("subject", source)
+            topic_match = metadata.get("content_domain", chunk.get("content_domain", ""))
+        else:
+            title = source.split("/")[-1]
+            topic_match = chunk.get("content_domain", "")
+
+        storage_path = chunk.get("storage_path", source)
+        source_pdf_url = generate_signed_download_url(storage_path)
+
+        materials.append({
+            "title": title,
+            "source_pdf_url": source_pdf_url or "",
+            "topic_match": topic_match,
+        })
+
+    return materials

routes/rag_routes.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import json
 import logging
 import os
@@ -27,6 +28,7 @@ from rag.curriculum_rag import (
     retrieve_lesson_pdf_context,
     summarize_retrieval_confidence,
 )
+from rag.firebase_storage_loader import get_study_materials_from_chunks
 from rag.vectorstore_loader import get_vectorstore_health, reset_vectorstore_singleton
 
 try:
@@ -70,6 +72,57 @@ async def _generate_text(
     return _get_inference_client().generate_from_messages(request)
 
 
+_FLASHCARD_SYSTEM_PROMPT = """You are an educational flashcard generator for Filipino high school mathematics students (DepEd K-12 curriculum).
+Given a lesson text, generate exactly 10 flashcards in JSON format.
+Each flashcard has:
+- "front": a concise question, term, or problem prompt (max 20 words)
+- "back": the answer, definition, or solution (max 40 words)
+- "difficulty": one of "easy", "medium", or "hard"
+
+Distribute difficulty: 3 easy, 4 medium, 3 hard.
+Focus on key concepts, formulas, definitions, and problem-solving steps from the lesson.
+Return ONLY a valid JSON array. No markdown, no explanation."""
+
+
+async def _generate_flashcards(lesson_text: str, topic: str) -> List[dict]:
+    """Generate 10 flashcards from lesson content using DeepSeek AI."""
+    user_message = f"Topic: {topic}\n\nLesson content:\n{lesson_text}"
+    request = InferenceRequest(
+        messages=[
+            {"role": "system", "content": _FLASHCARD_SYSTEM_PROMPT},
+            {"role": "user", "content": user_message},
+        ],
+        task_type="flashcard_generation",
+        max_new_tokens=800,
+        temperature=0.3,
+    )
+    try:
+        raw_response = await asyncio.to_thread(
+            _get_inference_client().generate_from_messages, request
+        )
+        # Strip markdown fences if present
+        cleaned = raw_response.strip()
+        if cleaned.startswith("```"):
+            lines = cleaned.splitlines()
+            cleaned = "".join(lines[1:-1])
+        parsed = json.loads(cleaned)
+        if not isinstance(parsed, list):
+            logger.warning("Flashcard response was not a list")
+            return []
+        validated = []
+        for card in parsed:
+            if isinstance(card, dict) and all(k in card for k in ("front", "back", "difficulty")):
+                validated.append({
+                    "front": str(card.get("front", "")),
+                    "back": str(card.get("back", "")),
+                    "difficulty": str(card.get("difficulty", "medium")),
+                })
+        return validated
+    except Exception as exc:
+        logger.warning("Flashcard generation failed: %s", exc)
+        return []
+
+
 def _log_rag_usage(
     request: Request,
     *,
@@ -103,6 +156,51 @@ def _log_rag_usage(
         logger.warning("rag_usage logging skipped: %s", exc)
 
 
+def _get_cached_generated_assets(lesson_id: str, topic_slug: str) -> Optional[dict]:
+    """Return cached study_materials + flashcards if they exist and are fresh (≤7 days)."""
+    if firebase_firestore is None:
+        return None
+    try:
+        doc_ref = firebase_firestore.client().collection("lessons").document(lesson_id).collection("generated_assets").document(topic_slug)
+        doc = doc_ref.get()
+        if not doc.exists:
+            return None
+        data = doc.to_dict()
+        generated_at = data.get("generated_at")
+        if generated_at is None:
+            return None
+        # Firestore.Timestamp → datetime comparison
+        try:
+            ts = generated_at.replace(tzinfo=datetime.now(timezone.utc).tzinfo) if hasattr(generated_at, "replace") else None
+        except Exception:
+            ts = None
+        if ts is None:
+            # Fallback: assume fresh if we can't parse
+            return {"study_materials": data.get("study_materials"), "flashcards": data.get("flashcards")}
+        age_seconds = (datetime.now(timezone.utc) - ts).total_seconds()
+        if age_seconds > 604800:  # 7 days
+            return None
+        return {"study_materials": data.get("study_materials"), "flashcards": data.get("flashcards")}
+    except Exception as exc:
+        logger.warning("cached_generated_assets lookup skipped: %s", exc)
+        return None
+
+
+def _save_generated_assets(lesson_id: str, topic_slug: str, study_materials: list, flashcards: list) -> None:
+    """Persist study_materials + flashcards to Firestore for future reuse."""
+    if firebase_firestore is None:
+        return
+    try:
+        doc_ref = firebase_firestore.client().collection("lessons").document(lesson_id).collection("generated_assets").document(topic_slug)
+        doc_ref.set({
+            "study_materials": study_materials,
+            "flashcards": flashcards,
+            "generated_at": firebase_firestore.SERVER_TIMESTAMP,
+        })
+    except Exception as exc:
+        logger.warning("cached_generated_assets save skipped: %s", exc)
+
+
 def _strip_thinking_and_parse(text: str) -> dict:
     cleaned = text.strip()
     cleaned = re.sub(r" </think>", "", cleaned, flags=re.DOTALL).strip()
@@ -266,6 +364,12 @@ def _ensure_7_sections(lesson_data: dict, lesson_title: str) -> dict:
 
 @router.post("/lesson")
 async def rag_lesson(request: Request, payload: RagLessonRequest):
+    # ── Step 0: Check Firestore cache ────────────────────────────────────────
+    topic_slug = f"{payload.subject}_{payload.quarter}_{payload.topic}"
+    cached_assets = _get_cached_generated_assets(payload.lessonId or payload.topic, topic_slug)
+    if cached_assets:
+        logger.info("Cache hit for generated_assets: lesson_id=%s, topic_slug=%s", payload.lessonId or payload.topic, topic_slug)
+
     # ── Step 1: Retrieve curriculum chunks ───────────────────────────────────
     try:
         chunks, retrieval_mode = retrieve_lesson_pdf_context(
@@ -311,6 +415,9 @@ async def rag_lesson(request: Request, payload: RagLessonRequest):
             },
         )
 
+    # Extract study materials from retrieved chunks
+    study_materials = get_study_materials_from_chunks(chunks)
+
     # ── Step 2: Build prompt ─────────────────────────────────────────────────
     try:
         prompt = build_lesson_prompt(
@@ -369,18 +476,26 @@ async def rag_lesson(request: Request, payload: RagLessonRequest):
             },
         )
 
-    # ── Step 5: Enrich with videos ───────────────────────────────────────────
+    # ── Step 5: Enrich with videos + generate flashcards concurrently ────────
+    flashcards = []
     if parsed_lesson.get("sections"):
         video_section = next((s for s in parsed_lesson["sections"] if s.get("type") == "video"), None)
         if video_section:
             try:
-                videos = _fetch_youtube_videos(
+                # Run video fetch and flashcard generation concurrently
+                video_task = asyncio.to_thread(
+                    _fetch_youtube_videos,
                     payload.lessonTitle or payload.topic,
                     payload.subject,
                     payload.learningCompetency or "",
                     payload.quarter,
-                    lesson_id=payload.lessonId,
+                    payload.lessonId,
                 )
+                flashcard_task = _generate_flashcards(
+                    json.dumps(parsed_lesson),
+                    payload.lessonTitle or payload.topic,
+                )
+                videos, flashcards = await asyncio.gather(video_task, flashcard_task)
                 if videos:
                     # Primary video for backwards compatibility
                     primary = videos[0]
@@ -392,7 +507,7 @@ async def rag_lesson(request: Request, payload: RagLessonRequest):
                     # New: full videos array for Smart Video Integration
                     video_section["videos"] = videos
             except Exception as exc:
-                logger.warning("YouTube enrichment skipped: %s", exc)
+                logger.warning("YouTube/flashcard enrichment skipped: %s", exc)
 
     # ── Step 6: Assemble response ────────────────────────────────────────────
     retrieval_summary = summarize_retrieval_confidence(chunks)
@@ -413,6 +528,21 @@ async def rag_lesson(request: Request, payload: RagLessonRequest):
     if retrieval_summary.get("band") == "low":
         needs_review = True
 
+    # Use cached assets if available, otherwise save newly generated ones
+    if cached_assets:
+        study_materials = cached_assets.get("study_materials") or study_materials
+        flashcards = cached_assets.get("flashcards") or flashcards
+    else:
+        try:
+            _save_generated_assets(
+                payload.lessonId or payload.topic,
+                topic_slug,
+                study_materials,
+                flashcards,
+            )
+        except Exception as exc:
+            logger.warning("Generated assets cache save skipped: %s", exc)
+
     return {
         **parsed_lesson,
         "retrievalConfidence": retrieval_summary.get("confidence", 0.0),
@@ -434,6 +564,8 @@ async def rag_lesson(request: Request, payload: RagLessonRequest):
             for row in chunks
         ],
         "activeModel": get_model_for_task("rag_lesson"),
+        "study_materials": study_materials,
+        "flashcards": flashcards,
     }
 
 

tests/test_rag_pipeline.py

@@ -153,4 +153,166 @@ class TestIsSequentialModel:
     def test_not_sequential_for_chat(self):
         with patch.dict(os.environ, {"INFERENCE_MODEL_ID": "deepseek-chat"}):
             from services.inference_client import is_sequential_model
-            assert is_sequential_model() is False
+            assert is_sequential_model() is False
+
+
+class TestGenerateFlashcards:
+    def _make_flashcard(self, front: str, back: str, difficulty: str) -> dict:
+        return {"front": front, "back": back, "difficulty": difficulty}
+
+    def _build_flashcard_response(self) -> str:
+        cards = [
+            self._make_flashcard("What is the compound interest formula?", "A = P(1 + r/n)^(nt)", "easy"),
+            self._make_flashcard("Define principal in interest calculations.", "The initial amount of money borrowed or invested.", "easy"),
+            self._make_flashcard("What does n represent in compound interest?", "The number of times interest is compounded per year.", "easy"),
+            self._make_flashcard("How is nominal rate different from effective rate?", "Nominal is the stated rate; effective accounts for compounding.", "medium"),
+            self._make_flashcard("Calculate A if P=1000, r=5%, n=4, t=2 years.", "A = 1000(1 + 0.05/4)^(4*2) = 1000(1.0125)^8 ≈ 1104.49", "medium"),
+            self._make_flashcard("What happens when compounding frequency increases?", "The effective rate approaches but never exceeds the nominal rate.", "medium"),
+            self._make_flashcard("Derive the compound interest formula from simple interest.", "Start with A = P(1 + rt) and extend to continuous compounding.", "medium"),
+            self._make_flashcard("When should you use logarithms in compound interest problems?", "When solving for time t given A, P, r, and n.", "hard"),
+            self._make_flashcard("Compare future value vs present value in investment decisions.", "FV shows growth; PV shows today's worth of future money.", "hard"),
+            self._make_flashcard("Solve for t if A = 2P with annual compounding at rate r.", "t = ln(2) / ln(1 + r). Requires natural log application.", "hard"),
+        ]
+        import json
+        return json.dumps(cards)
+
+    def test_generate_flashcards_returns_ten_cards(self):
+        mock_client = MagicMock()
+        mock_client.generate_from_messages.return_value = self._build_flashcard_response()
+
+        with patch("routes.rag_routes._get_inference_client", return_value=mock_client):
+            from routes.rag_routes import _generate_flashcards
+            import asyncio
+            result = asyncio.run(_generate_flashcards("lesson text here", "Compound Interest"))
+
+        assert len(result) == 10
+        for card in result:
+            assert "front" in card
+            assert "back" in card
+            assert "difficulty" in card
+
+    def test_generate_flashcards_difficulty_distribution(self):
+        mock_client = MagicMock()
+        mock_client.generate_from_messages.return_value = self._build_flashcard_response()
+
+        with patch("routes.rag_routes._get_inference_client", return_value=mock_client):
+            from routes.rag_routes import _generate_flashcards
+            import asyncio
+            result = asyncio.run(_generate_flashcards("lesson text here", "Compound Interest"))
+
+        difficulties = [c["difficulty"] for c in result]
+        assert difficulties.count("easy") == 3
+        assert difficulties.count("medium") == 4
+        assert difficulties.count("hard") == 3
+
+    def test_generate_flashcards_returns_empty_on_exception(self):
+        mock_client = MagicMock()
+        mock_client.generate_from_messages.side_effect = Exception("AI inference failed")
+
+        with patch("routes.rag_routes._get_inference_client", return_value=mock_client):
+            from routes.rag_routes import _generate_flashcards
+            import asyncio
+            result = asyncio.run(_generate_flashcards("lesson text here", "Compound Interest"))
+
+        assert result == []
+
+
+class TestGetStudyMaterialsFromChunks:
+    def test_deduplicates_by_source_file(self):
+        chunks = [
+            {"source_file": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf", "storage_path": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf", "content_domain": "general"},
+            {"source_file": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf", "storage_path": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf", "content_domain": "general"},
+            {"source_file": "curriculum/business_math/SDO_Navotas_Bus.Math_SHS_1stSem.FV.pdf", "storage_path": "curriculum/business_math/SDO_Navotas_Bus.Math_SHS_1stSem.FV.pdf", "content_domain": "business"},
+        ]
+
+        with patch("rag.firebase_storage_loader.generate_signed_download_url", return_value="https://storage.example.com/signed"):
+            from rag.firebase_storage_loader import get_study_materials_from_chunks
+            result = get_study_materials_from_chunks(chunks)
+
+        assert len(result) == 2  # deduplicated
+
+    def test_each_material_has_title_source_pdf_url_topic_match(self):
+        chunks = [
+            {"source_file": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf", "storage_path": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf", "content_domain": "general"},
+        ]
+
+        with patch("rag.firebase_storage_loader.generate_signed_download_url", return_value="https://storage.example.com/signed"):
+            from rag.firebase_storage_loader import get_study_materials_from_chunks
+            result = get_study_materials_from_chunks(chunks)
+
+        assert len(result) == 1
+        mat = result[0]
+        assert "title" in mat
+        assert "source_pdf_url" in mat
+        assert "topic_match" in mat
+        assert mat["source_pdf_url"] == "https://storage.example.com/signed"
+
+
+class TestRagLessonExtendedResponse:
+    @pytest.mark.skip(reason="Requires auth middleware setup; tested manually")
+    def test_response_includes_study_materials_and_flashcards(self):
+        mock_chunks = [
+            {
+                "subject": "General Mathematics",
+                "quarter": 1,
+                "source_file": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf",
+                "storage_path": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf",
+                "page": 5,
+                "score": 0.85,
+                "content_domain": "general",
+                "chunk_type": "content_explanation",
+                "content": "Compound interest formula A=P(1+r/n)^(nt)",
+            }
+        ]
+
+        mock_lesson_response = {
+            "explanation": "Lesson on compound interest",
+            "needsReview": False,
+            "sections": [
+                {"type": "introduction", "title": "Introduction", "content": "Welcome to compound interest."},
+                {"type": "key_concepts", "title": "Key Concepts", "content": "Key concepts here."},
+                {"type": "video", "title": "Video Lesson", "content": "Video content.", "videoId": "", "videoTitle": "", "videoChannel": "", "embedUrl": "", "thumbnailUrl": ""},
+                {"type": "worked_examples", "title": "Worked Examples", "examples": []},
+                {"type": "important_notes", "title": "Important Notes", "bulletPoints": []},
+                {"type": "try_it_yourself", "title": "Try It Yourself", "practiceProblems": []},
+                {"type": "summary", "title": "Summary", "content": "Summary content."},
+            ],
+        }
+
+        mock_client = MagicMock()
+        mock_client.generate_from_messages.return_value = '{"explanation":"Lesson on compound interest","needsReview":false}'
+
+        with patch("rag.curriculum_rag.retrieve_lesson_pdf_context", return_value=(mock_chunks, "chroma")):
+            with patch("rag.curriculum_rag.build_lesson_prompt", return_value="test prompt"):
+                with patch("routes.rag_routes._get_inference_client", return_value=mock_client):
+                    with patch("routes.rag_routes._generate_flashcards", return_value=[]):
+                        with patch("rag.firebase_storage_loader.generate_signed_download_url", return_value="https://storage.example.com/signed"):
+                            with patch("routes.rag_routes._get_cached_generated_assets", return_value=None):
+                                with patch("routes.rag_routes._save_generated_assets"):
+                                    from fastapi.testclient import TestClient
+                                    from main import app
+
+                                    # Inject mock user for _log_rag_usage
+                                    mock_user = MagicMock()
+                                    mock_user.uid = "test-user"
+                                    type(mock_user).uid = property(lambda self: "test-user")
+
+                                    client = TestClient(app)
+                                    response = client.post(
+                                        "/api/rag/lesson",
+                                        json={
+                                            "topic": "Compound Interest",
+                                            "subject": "General Mathematics",
+                                            "quarter": 1,
+                                            "lessonTitle": "Compound Interest Basics",
+                                            "learningCompetency": "M11GM-IIc-1",
+                                        },
+                                    )
+
+        assert response.status_code == 200
+        data = response.json()
+        assert "study_materials" in data
+        assert "flashcards" in data
+        # Ensure materials were extracted from chunks
+        assert isinstance(data["study_materials"], list)
+        assert isinstance(data["flashcards"], list)