🚀 Auto-deploy backend from GitHub (7fe6d9b)

rag/firebase_storage_loader.py

@@ -5,7 +5,6 @@ Downloads PDFs from Firebase Storage and extracts text for ChromaDB indexing.
 
 from __future__ import annotations
 
-import datetime
 import logging
 import os
 from pathlib import Path
@@ -149,72 +148,4 @@ PDF_METADATA: Dict[str, dict] = {
         "quarter": 1,
         "storage_path": "curriculum/stat_prob/SDO_Navotas_STAT_PROB_SHS_1stSem.FV.pdf",
     },
-}
-
-
-def generate_signed_download_url(storage_path: str, expiration_hours: int = 24) -> Optional[str]:
-    """Generate a signed download URL for a Firebase Storage blob.
-
-    Args:
-        storage_path: The path of the blob in Firebase Storage.
-        expiration_hours: Number of hours until the URL expires (default 24).
-
-    Returns:
-        Signed URL string, or None if Firebase Storage is unavailable.
-    """
-    _, bucket = _init_firebase_storage()
-    if bucket is None:
-        logger.warning("Firebase Storage not available, cannot generate signed URL")
-        return None
-
-    try:
-        blob = bucket.blob(storage_path)
-        signed_url = blob.generate_signed_url(
-            expiration=datetime.timedelta(hours=expiration_hours),
-            method="GET",
-        )
-        logger.info("Generated signed URL for %s (expires in %dh)", storage_path, expiration_hours)
-        return signed_url
-    except Exception as e:
-        logger.error("Failed to generate signed URL for %s: %s", storage_path, e)
-        return None
-
-
-def get_study_materials_from_chunks(chunks: list[dict]) -> list[dict]:
-    """Extract study materials from chunks, deduplicating by source PDF.
-
-    Args:
-        chunks: List of chunk dicts with optional `source_file`, `storage_path`,
-               and `content_domain` keys.
-
-    Returns:
-        List of dicts with keys: `title`, `source_pdf_url`, `topic_match`.
-    """
-    seen_sources: set[str] = set()
-    materials: list[dict] = []
-
-    for chunk in chunks:
-        source = chunk.get("source_file")
-        if not source or source in seen_sources:
-            continue
-        seen_sources.add(source)
-
-        # Look up PDF metadata by storage_path
-        metadata = PDF_METADATA.get(source)
-        if metadata:
-            title = metadata.get("subject", source)
-            topic_match = metadata.get("content_domain", chunk.get("content_domain", ""))
-        else:
-            title = source.split("/")[-1]
-            topic_match = chunk.get("content_domain", "")
-
-        storage_path = chunk.get("storage_path", source)
-        source_pdf_url = generate_signed_download_url(storage_path)
-
-        materials.append({
-            "title": title,
-            "source_pdf_url": source_pdf_url or "",
-            "topic_match": topic_match,
-        })
-
-    return materials
+}

routes/rag_routes.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import asyncio
 import json
 import logging
 import os
@@ -28,7 +27,6 @@ from rag.curriculum_rag import (
     retrieve_lesson_pdf_context,
     summarize_retrieval_confidence,
 )
-from rag.firebase_storage_loader import get_study_materials_from_chunks
 from rag.vectorstore_loader import get_vectorstore_health, reset_vectorstore_singleton
 
 try:
@@ -72,57 +70,6 @@ async def _generate_text(
     return _get_inference_client().generate_from_messages(request)
 
 
-_FLASHCARD_SYSTEM_PROMPT = """You are an educational flashcard generator for Filipino high school mathematics students (DepEd K-12 curriculum).
-Given a lesson text, generate exactly 10 flashcards in JSON format.
-Each flashcard has:
-- "front": a concise question, term, or problem prompt (max 20 words)
-- "back": the answer, definition, or solution (max 40 words)
-- "difficulty": one of "easy", "medium", or "hard"
-
-Distribute difficulty: 3 easy, 4 medium, 3 hard.
-Focus on key concepts, formulas, definitions, and problem-solving steps from the lesson.
-Return ONLY a valid JSON array. No markdown, no explanation."""
-
-
-async def _generate_flashcards(lesson_text: str, topic: str) -> List[dict]:
-    """Generate 10 flashcards from lesson content using DeepSeek AI."""
-    user_message = f"Topic: {topic}\n\nLesson content:\n{lesson_text}"
-    request = InferenceRequest(
-        messages=[
-            {"role": "system", "content": _FLASHCARD_SYSTEM_PROMPT},
-            {"role": "user", "content": user_message},
-        ],
-        task_type="flashcard_generation",
-        max_new_tokens=800,
-        temperature=0.3,
-    )
-    try:
-        raw_response = await asyncio.to_thread(
-            _get_inference_client().generate_from_messages, request
-        )
-        # Strip markdown fences if present
-        cleaned = raw_response.strip()
-        if cleaned.startswith("```"):
-            lines = cleaned.splitlines()
-            cleaned = "".join(lines[1:-1])
-        parsed = json.loads(cleaned)
-        if not isinstance(parsed, list):
-            logger.warning("Flashcard response was not a list")
-            return []
-        validated = []
-        for card in parsed:
-            if isinstance(card, dict) and all(k in card for k in ("front", "back", "difficulty")):
-                validated.append({
-                    "front": str(card.get("front", "")),
-                    "back": str(card.get("back", "")),
-                    "difficulty": str(card.get("difficulty", "medium")),
-                })
-        return validated
-    except Exception as exc:
-        logger.warning("Flashcard generation failed: %s", exc)
-        return []
-
-
 def _log_rag_usage(
     request: Request,
     *,
@@ -156,51 +103,6 @@ def _log_rag_usage(
         logger.warning("rag_usage logging skipped: %s", exc)
 
 
-def _get_cached_generated_assets(lesson_id: str, topic_slug: str) -> Optional[dict]:
-    """Return cached study_materials + flashcards if they exist and are fresh (≤7 days)."""
-    if firebase_firestore is None:
-        return None
-    try:
-        doc_ref = firebase_firestore.client().collection("lessons").document(lesson_id).collection("generated_assets").document(topic_slug)
-        doc = doc_ref.get()
-        if not doc.exists:
-            return None
-        data = doc.to_dict()
-        generated_at = data.get("generated_at")
-        if generated_at is None:
-            return None
-        # Firestore.Timestamp → datetime comparison
-        try:
-            ts = generated_at.replace(tzinfo=datetime.now(timezone.utc).tzinfo) if hasattr(generated_at, "replace") else None
-        except Exception:
-            ts = None
-        if ts is None:
-            # Fallback: assume fresh if we can't parse
-            return {"study_materials": data.get("study_materials"), "flashcards": data.get("flashcards")}
-        age_seconds = (datetime.now(timezone.utc) - ts).total_seconds()
-        if age_seconds > 604800:  # 7 days
-            return None
-        return {"study_materials": data.get("study_materials"), "flashcards": data.get("flashcards")}
-    except Exception as exc:
-        logger.warning("cached_generated_assets lookup skipped: %s", exc)
-        return None
-
-
-def _save_generated_assets(lesson_id: str, topic_slug: str, study_materials: list, flashcards: list) -> None:
-    """Persist study_materials + flashcards to Firestore for future reuse."""
-    if firebase_firestore is None:
-        return
-    try:
-        doc_ref = firebase_firestore.client().collection("lessons").document(lesson_id).collection("generated_assets").document(topic_slug)
-        doc_ref.set({
-            "study_materials": study_materials,
-            "flashcards": flashcards,
-            "generated_at": firebase_firestore.SERVER_TIMESTAMP,
-        })
-    except Exception as exc:
-        logger.warning("cached_generated_assets save skipped: %s", exc)
-
-
 def _strip_thinking_and_parse(text: str) -> dict:
     cleaned = text.strip()
     cleaned = re.sub(r" </think>", "", cleaned, flags=re.DOTALL).strip()
@@ -364,12 +266,6 @@ def _ensure_7_sections(lesson_data: dict, lesson_title: str) -> dict:
 
 @router.post("/lesson")
 async def rag_lesson(request: Request, payload: RagLessonRequest):
-    # ── Step 0: Check Firestore cache ────────────────────────────────────────
-    topic_slug = f"{payload.subject}_{payload.quarter}_{payload.topic}"
-    cached_assets = _get_cached_generated_assets(payload.lessonId or payload.topic, topic_slug)
-    if cached_assets:
-        logger.info("Cache hit for generated_assets: lesson_id=%s, topic_slug=%s", payload.lessonId or payload.topic, topic_slug)
-
     # ── Step 1: Retrieve curriculum chunks ───────────────────────────────────
     try:
         chunks, retrieval_mode = retrieve_lesson_pdf_context(
@@ -415,9 +311,6 @@ async def rag_lesson(request: Request, payload: RagLessonRequest):
             },
         )
 
-    # Extract study materials from retrieved chunks
-    study_materials = get_study_materials_from_chunks(chunks)
-
     # ── Step 2: Build prompt ─────────────────────────────────────────────────
     try:
         prompt = build_lesson_prompt(
@@ -476,26 +369,18 @@ async def rag_lesson(request: Request, payload: RagLessonRequest):
             },
         )
 
-    # ── Step 5: Enrich with videos + generate flashcards concurrently ────────
-    flashcards = []
+    # ── Step 5: Enrich with videos ───────────────────────────────────────────
     if parsed_lesson.get("sections"):
         video_section = next((s for s in parsed_lesson["sections"] if s.get("type") == "video"), None)
         if video_section:
             try:
-                # Run video fetch and flashcard generation concurrently
-                video_task = asyncio.to_thread(
-                    _fetch_youtube_videos,
+                videos = _fetch_youtube_videos(
                     payload.lessonTitle or payload.topic,
                     payload.subject,
                     payload.learningCompetency or "",
                     payload.quarter,
-                    payload.lessonId,
+                    lesson_id=payload.lessonId,
                 )
-                flashcard_task = _generate_flashcards(
-                    json.dumps(parsed_lesson),
-                    payload.lessonTitle or payload.topic,
-                )
-                videos, flashcards = await asyncio.gather(video_task, flashcard_task)
                 if videos:
                     # Primary video for backwards compatibility
                     primary = videos[0]
@@ -507,7 +392,7 @@ async def rag_lesson(request: Request, payload: RagLessonRequest):
                     # New: full videos array for Smart Video Integration
                     video_section["videos"] = videos
             except Exception as exc:
-                logger.warning("YouTube/flashcard enrichment skipped: %s", exc)
+                logger.warning("YouTube enrichment skipped: %s", exc)
 
     # ── Step 6: Assemble response ────────────────────────────────────────────
     retrieval_summary = summarize_retrieval_confidence(chunks)
@@ -528,21 +413,6 @@ async def rag_lesson(request: Request, payload: RagLessonRequest):
     if retrieval_summary.get("band") == "low":
         needs_review = True
 
-    # Use cached assets if available, otherwise save newly generated ones
-    if cached_assets:
-        study_materials = cached_assets.get("study_materials") or study_materials
-        flashcards = cached_assets.get("flashcards") or flashcards
-    else:
-        try:
-            _save_generated_assets(
-                payload.lessonId or payload.topic,
-                topic_slug,
-                study_materials,
-                flashcards,
-            )
-        except Exception as exc:
-            logger.warning("Generated assets cache save skipped: %s", exc)
-
     return {
         **parsed_lesson,
         "retrievalConfidence": retrieval_summary.get("confidence", 0.0),
@@ -564,8 +434,6 @@ async def rag_lesson(request: Request, payload: RagLessonRequest):
             for row in chunks
         ],
         "activeModel": get_model_for_task("rag_lesson"),
-        "study_materials": study_materials,
-        "flashcards": flashcards,
     }
 
 

tests/test_rag_pipeline.py

@@ -153,166 +153,4 @@ class TestIsSequentialModel:
     def test_not_sequential_for_chat(self):
         with patch.dict(os.environ, {"INFERENCE_MODEL_ID": "deepseek-chat"}):
             from services.inference_client import is_sequential_model
-            assert is_sequential_model() is False
-
-
-class TestGenerateFlashcards:
-    def _make_flashcard(self, front: str, back: str, difficulty: str) -> dict:
-        return {"front": front, "back": back, "difficulty": difficulty}
-
-    def _build_flashcard_response(self) -> str:
-        cards = [
-            self._make_flashcard("What is the compound interest formula?", "A = P(1 + r/n)^(nt)", "easy"),
-            self._make_flashcard("Define principal in interest calculations.", "The initial amount of money borrowed or invested.", "easy"),
-            self._make_flashcard("What does n represent in compound interest?", "The number of times interest is compounded per year.", "easy"),
-            self._make_flashcard("How is nominal rate different from effective rate?", "Nominal is the stated rate; effective accounts for compounding.", "medium"),
-            self._make_flashcard("Calculate A if P=1000, r=5%, n=4, t=2 years.", "A = 1000(1 + 0.05/4)^(4*2) = 1000(1.0125)^8 ≈ 1104.49", "medium"),
-            self._make_flashcard("What happens when compounding frequency increases?", "The effective rate approaches but never exceeds the nominal rate.", "medium"),
-            self._make_flashcard("Derive the compound interest formula from simple interest.", "Start with A = P(1 + rt) and extend to continuous compounding.", "medium"),
-            self._make_flashcard("When should you use logarithms in compound interest problems?", "When solving for time t given A, P, r, and n.", "hard"),
-            self._make_flashcard("Compare future value vs present value in investment decisions.", "FV shows growth; PV shows today's worth of future money.", "hard"),
-            self._make_flashcard("Solve for t if A = 2P with annual compounding at rate r.", "t = ln(2) / ln(1 + r). Requires natural log application.", "hard"),
-        ]
-        import json
-        return json.dumps(cards)
-
-    def test_generate_flashcards_returns_ten_cards(self):
-        mock_client = MagicMock()
-        mock_client.generate_from_messages.return_value = self._build_flashcard_response()
-
-        with patch("routes.rag_routes._get_inference_client", return_value=mock_client):
-            from routes.rag_routes import _generate_flashcards
-            import asyncio
-            result = asyncio.run(_generate_flashcards("lesson text here", "Compound Interest"))
-
-        assert len(result) == 10
-        for card in result:
-            assert "front" in card
-            assert "back" in card
-            assert "difficulty" in card
-
-    def test_generate_flashcards_difficulty_distribution(self):
-        mock_client = MagicMock()
-        mock_client.generate_from_messages.return_value = self._build_flashcard_response()
-
-        with patch("routes.rag_routes._get_inference_client", return_value=mock_client):
-            from routes.rag_routes import _generate_flashcards
-            import asyncio
-            result = asyncio.run(_generate_flashcards("lesson text here", "Compound Interest"))
-
-        difficulties = [c["difficulty"] for c in result]
-        assert difficulties.count("easy") == 3
-        assert difficulties.count("medium") == 4
-        assert difficulties.count("hard") == 3
-
-    def test_generate_flashcards_returns_empty_on_exception(self):
-        mock_client = MagicMock()
-        mock_client.generate_from_messages.side_effect = Exception("AI inference failed")
-
-        with patch("routes.rag_routes._get_inference_client", return_value=mock_client):
-            from routes.rag_routes import _generate_flashcards
-            import asyncio
-            result = asyncio.run(_generate_flashcards("lesson text here", "Compound Interest"))
-
-        assert result == []
-
-
-class TestGetStudyMaterialsFromChunks:
-    def test_deduplicates_by_source_file(self):
-        chunks = [
-            {"source_file": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf", "storage_path": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf", "content_domain": "general"},
-            {"source_file": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf", "storage_path": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf", "content_domain": "general"},
-            {"source_file": "curriculum/business_math/SDO_Navotas_Bus.Math_SHS_1stSem.FV.pdf", "storage_path": "curriculum/business_math/SDO_Navotas_Bus.Math_SHS_1stSem.FV.pdf", "content_domain": "business"},
-        ]
-
-        with patch("rag.firebase_storage_loader.generate_signed_download_url", return_value="https://storage.example.com/signed"):
-            from rag.firebase_storage_loader import get_study_materials_from_chunks
-            result = get_study_materials_from_chunks(chunks)
-
-        assert len(result) == 2  # deduplicated
-
-    def test_each_material_has_title_source_pdf_url_topic_match(self):
-        chunks = [
-            {"source_file": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf", "storage_path": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf", "content_domain": "general"},
-        ]
-
-        with patch("rag.firebase_storage_loader.generate_signed_download_url", return_value="https://storage.example.com/signed"):
-            from rag.firebase_storage_loader import get_study_materials_from_chunks
-            result = get_study_materials_from_chunks(chunks)
-
-        assert len(result) == 1
-        mat = result[0]
-        assert "title" in mat
-        assert "source_pdf_url" in mat
-        assert "topic_match" in mat
-        assert mat["source_pdf_url"] == "https://storage.example.com/signed"
-
-
-class TestRagLessonExtendedResponse:
-    @pytest.mark.skip(reason="Requires auth middleware setup; tested manually")
-    def test_response_includes_study_materials_and_flashcards(self):
-        mock_chunks = [
-            {
-                "subject": "General Mathematics",
-                "quarter": 1,
-                "source_file": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf",
-                "storage_path": "curriculum/gen_math_sdo/SDO_Navotas_Gen.Math_SHS_1stSem.FV.pdf",
-                "page": 5,
-                "score": 0.85,
-                "content_domain": "general",
-                "chunk_type": "content_explanation",
-                "content": "Compound interest formula A=P(1+r/n)^(nt)",
-            }
-        ]
-
-        mock_lesson_response = {
-            "explanation": "Lesson on compound interest",
-            "needsReview": False,
-            "sections": [
-                {"type": "introduction", "title": "Introduction", "content": "Welcome to compound interest."},
-                {"type": "key_concepts", "title": "Key Concepts", "content": "Key concepts here."},
-                {"type": "video", "title": "Video Lesson", "content": "Video content.", "videoId": "", "videoTitle": "", "videoChannel": "", "embedUrl": "", "thumbnailUrl": ""},
-                {"type": "worked_examples", "title": "Worked Examples", "examples": []},
-                {"type": "important_notes", "title": "Important Notes", "bulletPoints": []},
-                {"type": "try_it_yourself", "title": "Try It Yourself", "practiceProblems": []},
-                {"type": "summary", "title": "Summary", "content": "Summary content."},
-            ],
-        }
-
-        mock_client = MagicMock()
-        mock_client.generate_from_messages.return_value = '{"explanation":"Lesson on compound interest","needsReview":false}'
-
-        with patch("rag.curriculum_rag.retrieve_lesson_pdf_context", return_value=(mock_chunks, "chroma")):
-            with patch("rag.curriculum_rag.build_lesson_prompt", return_value="test prompt"):
-                with patch("routes.rag_routes._get_inference_client", return_value=mock_client):
-                    with patch("routes.rag_routes._generate_flashcards", return_value=[]):
-                        with patch("rag.firebase_storage_loader.generate_signed_download_url", return_value="https://storage.example.com/signed"):
-                            with patch("routes.rag_routes._get_cached_generated_assets", return_value=None):
-                                with patch("routes.rag_routes._save_generated_assets"):
-                                    from fastapi.testclient import TestClient
-                                    from main import app
-
-                                    # Inject mock user for _log_rag_usage
-                                    mock_user = MagicMock()
-                                    mock_user.uid = "test-user"
-                                    type(mock_user).uid = property(lambda self: "test-user")
-
-                                    client = TestClient(app)
-                                    response = client.post(
-                                        "/api/rag/lesson",
-                                        json={
-                                            "topic": "Compound Interest",
-                                            "subject": "General Mathematics",
-                                            "quarter": 1,
-                                            "lessonTitle": "Compound Interest Basics",
-                                            "learningCompetency": "M11GM-IIc-1",
-                                        },
-                                    )
-
-        assert response.status_code == 200
-        data = response.json()
-        assert "study_materials" in data
-        assert "flashcards" in data
-        # Ensure materials were extracted from chunks
-        assert isinstance(data["study_materials"], list)
-        assert isinstance(data["flashcards"], list)
+            assert is_sequential_model() is False