feat: implement summarization engine with Pydantic schemas and map-reduce processing pipeline

src/categorization/topic_classifier.py

@@ -9,9 +9,9 @@ Usage:
     # => "Technology & AI"
 
 Categories:
-    Technology & AI | Business & Finance | Education & Science
-    Productivity & Self-Growth | News & Politics
-    Entertainment & Lifestyle | Health & Sports
+    Technology & AI | Business & Finance | Education | Science
+    Productivity & Self-Growth | Health & Wellness | Sports & Fitness
+    Entertainment | History | Philosophy | Arts & Culture
 """
 
 from typing import List, Set
@@ -28,11 +28,15 @@ logger = setup_logger(__name__)
 CATEGORIES = [
     "Technology & AI",
     "Business & Finance",
-    "Education & Science",
+    "Education",
+    "Science",
     "Productivity & Self-Growth",
-    "News & Politics",
-    "Entertainment & Lifestyle",
-    "Health & Sports",
+    "Health & Wellness",
+    "Sports & Fitness",
+    "Entertainment",
+    "History",
+    "Philosophy",
+    "Arts & Culture",
 ]
 
 
@@ -98,12 +102,22 @@ _register("Business & Finance", [
     "ربح", "دخل", "ميزانية",
 ])
 
-# ── Education & Science ──
-_register("Education & Science", [
+# ── Education ──
+_register("Education", [
     # English
     "education", "learning", "teaching", "school", "university", "college",
-    "academic", "research", "study", "studying", "exam", "exams", "course",
+    "academic", "study", "studying", "exam", "exams", "course",
     "tutorial", "lecture", "scholarship", "degree", "phd", "thesis",
+    "curriculum", "pedagogy", "classroom", "student", "teacher",
+    "grammar", "language", "linguistics",
+    # Arabic
+    "تعليم", "تعلم", "مدرسة", "جامعة", "دراسة", "امتحان", "منهج",
+    "محاضرة", "طالب", "معلم",
+])
+
+# ── Science ──
+_register("Science", [
+    # English
     "science", "physics", "chemistry", "biology", "math", "mathematics",
     "statistics", "calculus", "algebra", "geometry", "astronomy", "space",
     "nasa", "quantum", "quantum physics", "quantum computing",
@@ -111,12 +125,10 @@ _register("Education & Science", [
     "climate", "climate change", "environment", "engineering",
     "mechanical engineering", "electrical engineering", "civil engineering",
     "experiment", "laboratory", "lab", "hypothesis", "theory",
-    "history", "philosophy", "psychology", "sociology", "linguistics",
-    "anthropology", "archaeology", "literature", "language", "grammar",
+    "research", "psychology", "sociology", "anthropology",
     # Arabic
-    "تعليم", "تعلم", "مدرسة", "جامعة", "علوم", "فيزياء", "كيمياء",
-    "أحياء", "رياضيات", "بحث", "دراسة", "امتحان", "منهج", "محاضرة",
-    "هندسة", "تاريخ", "فلسفة", "علم نفس", "فلك", "بيئة",
+    "علوم", "فيزياء", "كيمياء", "أحياء", "رياضيات", "بحث",
+    "هندسة", "فلك", "بيئة", "علم نفس",
 ])
 
 # ── Productivity & Self-Growth ──
@@ -128,74 +140,92 @@ _register("Productivity & Self-Growth", [
     "mindset", "focus", "concentration", "efficiency", "organization",
     "planning", "journaling", "morning routine", "routine", "success",
     "self help", "self-help", "life coaching", "coaching", "mentoring",
-    "mentor", "stoicism", "minimalism", "mindfulness", "meditation",
+    "mentor", "minimalism", "mindfulness",
     "emotional intelligence", "communication skills", "public speaking",
     "negotiation", "critical thinking", "problem solving", "creativity",
     "decision making", "confidence", "resilience", "work-life balance",
     "burnout", "career", "career development", "skill building",
     # Arabic
     "إنتاجية", "تطوير ذات", "تحفيز", "عادات", "إدارة الوقت",
-    "أهداف", "تركيز", "نجاح", "تخطيط", "تأمل", "ثقة بالنفس",
+    "أهداف", "تركيز", "نجاح", "تخطيط", "ثقة بالنفس",
     "مهارات", "تفكير", "إبداع",
 ])
 
-# ── News & Politics ──
-_register("News & Politics", [
+# ── Health & Wellness ──
+_register("Health & Wellness", [
+    # English
+    "health", "wellness", "mental health", "therapy", "depression",
+    "anxiety", "stress", "sleep", "yoga", "pilates", "meditation",
+    "diet", "nutrition", "calories", "protein", "vitamins",
+    "supplements", "weight loss", "fitness",
+    "medicine", "medical", "doctor", "hospital", "surgery", "disease",
+    "virus", "vaccine", "pandemic", "covid", "cancer", "diabetes",
+    "heart", "cardio", "physical therapy", "rehabilitation",
+    "first aid", "pharmacy", "drug", "prescription",
+    # Arabic
+    "صحة", "تغذية", "حمية", "صحة نفسية", "علاج", "طب",
+    "مستشفى", "نوم", "فيتامينات", "يوغا",
+])
+
+# ── Sports & Fitness ──
+_register("Sports & Fitness", [
     # English
-    "news", "politics", "political", "government", "policy", "election",
-    "elections", "democracy", "geopolitics", "diplomacy", "war", "conflict",
-    "military", "defense", "law", "legal", "legislation", "regulation",
-    "human rights", "immigration", "refugee", "sanctions", "united nations",
-    "nato", "eu", "european union", "congress", "parliament", "senate",
-    "president", "prime minister", "foreign policy", "domestic policy",
-    "protest", "activism", "corruption", "media", "journalism",
-    "press", "freedom of speech", "censorship", "propaganda",
-    "international relations", "treaty", "nuclear",
+    "sports", "football", "soccer", "basketball", "tennis", "baseball",
+    "cricket", "rugby", "boxing", "mma", "ufc", "wrestling",
+    "olympics", "world cup", "premier league", "nba", "nfl",
+    "exercise", "workout", "gym", "bodybuilding", "crossfit",
+    "running", "marathon", "swimming", "cycling", "hiking",
     # Arabic
-    "أخبار", "سياسة", "حكومة", "انتخابات", "ديمقراطية", "حرب",
-    "قانون", "حقوق إنسان", "دبلوماسية", "برلمان", "رئيس",
-    "إعلام", "صحافة",
+    "رياضة", "تمارين", "لياقة", "كرة قدم", "سباحة",
 ])
 
-# ── Entertainment & Lifestyle ──
-_register("Entertainment & Lifestyle", [
+# ── Entertainment ──
+_register("Entertainment", [
     # English
     "entertainment", "movie", "movies", "film", "films", "cinema",
     "tv", "television", "series", "netflix", "streaming", "anime",
     "manga", "gaming", "video games", "esports", "twitch", "youtube",
-    "podcast", "music", "song", "album", "concert", "artist",
-    "celebrity", "fashion", "style", "beauty", "makeup", "skincare",
-    "travel", "tourism", "food", "cooking", "recipe", "restaurant",
-    "cuisine", "vlog", "vlogging", "photography", "art", "design",
-    "graphic design", "illustration", "architecture", "interior design",
-    "diy", "crafts", "comedy", "humor", "drama", "reality tv",
+    "podcast", "music", "song", "album", "concert",
+    "celebrity", "comedy", "humor", "drama", "reality tv",
     "social media", "tiktok", "instagram", "influencer", "content creator",
-    "lifestyle", "luxury", "culture", "pop culture",
+    "vlog", "vlogging", "pop culture",
     # Arabic
     "ترفيه", "أفلام", "سينما", "مسلسلات", "ألعاب", "موسيقى",
-    "سفر", "طبخ", "أزياء", "جمال", "تصوير", "فن", "تصميم",
-    "ثقافة", "كوميديا", "يوتيوب",
+    "كوميديا", "يوتيوب",
 ])
 
-# ── Health & Sports ──
-_register("Health & Sports", [
+# ── History ──
+_register("History", [
     # English
-    "health", "fitness", "exercise", "workout", "gym", "bodybuilding",
-    "weight loss", "diet", "nutrition", "calories", "protein", "vitamins",
-    "supplements", "wellness", "mental health", "therapy", "depression",
-    "anxiety", "stress", "sleep", "yoga", "pilates", "crossfit",
-    "running", "marathon", "swimming", "cycling", "hiking",
-    "sports", "football", "soccer", "basketball", "tennis", "baseball",
-    "cricket", "rugby", "boxing", "mma", "ufc", "wrestling",
-    "olympics", "world cup", "premier league", "nba", "nfl",
-    "medicine", "medical", "doctor", "hospital", "surgery", "disease",
-    "virus", "vaccine", "pandemic", "covid", "cancer", "diabetes",
-    "heart", "cardio", "physical therapy", "rehabilitation",
-    "first aid", "pharmacy", "drug", "prescription",
+    "history", "ancient", "medieval", "civilization", "empire",
+    "world war", "revolution", "archaeology", "historical",
+    "dynasty", "colonialism", "independence", "heritage",
+    # Arabic
+    "تاريخ", "حضارة", "إمبراطورية", "ثورة", "آثار", "تراث",
+])
+
+# ── Philosophy ──
+_register("Philosophy", [
+    # English
+    "philosophy", "ethics", "morality", "existentialism", "stoicism",
+    "metaphysics", "epistemology", "logic", "consciousness",
+    "free will", "determinism", "nihilism", "virtue",
+    # Arabic
+    "فلسفة", "أخلاق", "وجودية", "منطق", "وعي",
+])
+
+# ── Arts & Culture ──
+_register("Arts & Culture", [
+    # English
+    "art", "artist", "painting", "sculpture", "gallery", "museum",
+    "photography", "design", "graphic design", "illustration",
+    "architecture", "interior design", "fashion", "style", "beauty",
+    "makeup", "skincare", "travel", "tourism", "food", "cooking",
+    "recipe", "restaurant", "cuisine", "diy", "crafts",
+    "culture", "literature", "lifestyle", "luxury",
     # Arabic
-    "صحة", "رياضة", "تمارين", "لياقة", "تغذية", "حمية",
-    "صحة نفسية", "علاج", "طب", "مستشفى", "كرة قدم", "سباحة",
-    "يوغا", "نوم", "فيتامينات",
+    "فن", "تصميم", "تصوير", "سفر", "طبخ", "أزياء",
+    "جمال", "ثقافة",
 ])
 
 
@@ -239,14 +269,14 @@ def classify_topic(topics: List[str]) -> str:
                 break
 
     if not matched:
-        return "Education & Science"
+        return "Education"
 
     # Return the first match in CATEGORIES order for consistency
     for cat in CATEGORIES:
         if cat in matched:
             return cat
 
-    return "Education & Science"
+    return "Education"
 
 
 def classify_topics(topics: List[str]) -> List[str]:
@@ -269,7 +299,7 @@ def classify_topic_groq(title: str, summary: str) -> str:
     Bypasses the local Zero-Shot classification model entirely.
     """
     if not title and not summary:
-        return "Education & Science"
+        return "Education"
 
     try:
         from src.utils.model_loader import get_groq_client
@@ -319,11 +349,11 @@ def classify_topic_groq(title: str, summary: str) -> str:
                 return cat
 
         logger.warning("⚠️ Groq returned invalid category: %s — falling back", reply)
-        return "Education & Science"
+        return "Education"
 
     except Exception as e:
         logger.error("❌ Groq category classification failed: %s", e, exc_info=True)
-        return "Education & Science"
+        return "Education"
 
 
 # ─────────────────────────────────────────────────────────────────────────────
@@ -341,7 +371,7 @@ def classify_topic_zeroshot(text: str) -> str:
         The best-matching category string from CATEGORIES.
     """
     if not text or not text.strip():
-        return "Education & Science"
+        return "Education"
 
     try:
         from src.utils.model_loader import get_classifier_pipeline
@@ -364,14 +394,14 @@ def classify_topic_zeroshot(text: str) -> str:
 
     except Exception as e:
         logger.warning("⚠️ Zero-shot classification failed: %s — falling back", e)
-        return "Education & Science"
+        return "Education"
 
 
 def classify_topic_hybrid(topics: List[str], text: str = "") -> str:
     """Best-of-both-worlds classifier.
 
     1. First tries fast keyword matching via ``classify_topic(topics)``.
-    2. If the result is the generic fallback ("Education & Science") AND
+    2. If the result is the generic fallback ("Education") AND
        ``text`` is provided, runs the mDeBERTa zero-shot classifier on
        the text for a more nuanced result.
 
@@ -385,7 +415,7 @@ def classify_topic_hybrid(topics: List[str], text: str = "") -> str:
     keyword_result = classify_topic(topics)
 
     # If keyword matching gave a confident answer, use it
-    if keyword_result != "Education & Science":
+    if keyword_result != "Education":
         return keyword_result
 
     # If we have text, try zero-shot as a fallback

src/summarization/note_generator.py

@@ -1,3 +1,4 @@
+import json
 import os
 import re
 from typing import Dict, List
@@ -34,51 +35,6 @@ _LANG_MATCH_INSTRUCTION = (
     "Identify the primary language of the input text. You MUST generate your entire response strictly in that exact same language. DO NOT mix languages under any circumstances."
 )
 
-def parse_key_points_from_markdown(markdown_text: str) -> List[str]:
-    """Extract bullet points under the Key Points header in Markdown."""
-    if not markdown_text:
-        return []
-
-    lines = markdown_text.splitlines()
-    key_points = []
-    in_key_points_section = False
-
-    for line in lines:
-        line_strip = line.strip()
-        if not line_strip:
-            continue
-        # Check for headers like ## 💡 Key Points or ## 💡 أهم النقاط or ## Key Points
-        if line_strip.startswith("##") and ("Key Points" in line_strip or "النقاط" in line_strip):
-            in_key_points_section = True
-            continue
-        elif line_strip.startswith("##") and in_key_points_section:
-            # Reached a new section header, stop parsing
-            break
-
-        if in_key_points_section:
-            # Match list items starting with -, *, or numbering
-            if line_strip.startswith("-") or line_strip.startswith("*"):
-                cleaned_point = line_strip.lstrip("-* ").strip()
-                if cleaned_point:
-                    key_points.append(cleaned_point)
-            elif line_strip[0].isdigit() and (line_strip[1] == "." or (len(line_strip) > 2 and line_strip[2] == ".")):
-                parts = line_strip.split(".", 1)
-                if len(parts) > 1:
-                    cleaned_point = parts[1].strip()
-                    if cleaned_point:
-                        key_points.append(cleaned_point)
-
-    # Fallback if parsing failed or section wasn't found
-    if not key_points:
-        for line in lines:
-            line_strip = line.strip()
-            if line_strip.startswith("- ") or line_strip.startswith("* "):
-                cleaned = line_strip[2:].strip()
-                if cleaned and not any(h in cleaned for h in ["General Summary", "الملخص العام", "Key Questions", "الأسئلة", "Key Points", "النقاط"]):
-                    key_points.append(cleaned)
-
-    return key_points[:5]
-
 
 
 
@@ -248,51 +204,153 @@ class NoteGenerator:
             )
         return segments_list
 
-    def _reduce_summary(self, segments_list: List[Dict], video_title: str) -> str:
-        """REDUCE step: summarize the combined chunk-summaries into one overall summary."""
+    # ── Valid categories for the reduce prompt ────────────────────────────
+    VALID_CATEGORIES = [
+        "Technology & AI",
+        "Business & Finance",
+        "Education",
+        "Science",
+        "Productivity & Self-Growth",
+        "Health & Wellness",
+        "Sports & Fitness",
+        "Entertainment",
+        "History",
+        "Philosophy",
+        "Arts & Culture",
+    ]
+
+    def _parse_reduce_json(self, raw_text: str, video_title: str) -> Dict:
+        """Safely parse the strict JSON returned by the reduce LLM call.
+
+        Returns a dict with keys: markdown_summary, key_points, category, language.
+        Falls back gracefully if parsing fails.
+        """
+        # Try to extract JSON from the response (handle markdown code fences)
+        text = raw_text.strip()
+        if text.startswith("```"):
+            # Remove ```json ... ``` wrappers
+            text = re.sub(r"^```(?:json)?\s*", "", text)
+            text = re.sub(r"\s*```$", "", text)
+
+        try:
+            data = json.loads(text)
+        except json.JSONDecodeError:
+            logger.warning("⚠️ Failed to parse reduce JSON. Falling back to raw text.")
+            return {
+                "markdown_summary": raw_text,
+                "key_points": [],
+                "category": "Education",
+                "language": "ar",
+            }
+
+        # Validate and sanitize each field
+        markdown_summary = data.get("markdown_summary", "").strip()
+        if not markdown_summary:
+            markdown_summary = raw_text
+
+        key_points = data.get("key_points", [])
+        if not isinstance(key_points, list):
+            key_points = []
+        key_points = [str(p).strip() for p in key_points if str(p).strip()][:5]
+
+        category = data.get("category", "").strip()
+        if category not in self.VALID_CATEGORIES:
+            # Attempt fuzzy match
+            category_lower = category.lower()
+            matched = False
+            for valid_cat in self.VALID_CATEGORIES:
+                if valid_cat.lower() in category_lower or category_lower in valid_cat.lower():
+                    category = valid_cat
+                    matched = True
+                    break
+            if not matched:
+                category = "Education"
+
+        language = data.get("language", "ar").strip().lower()
+        if language not in ("ar", "en"):
+            language = "ar"
+
+        return {
+            "markdown_summary": markdown_summary,
+            "key_points": key_points,
+            "category": category,
+            "language": language,
+        }
+
+    def _reduce_summary(self, segments_list: List[Dict], video_title: str) -> Dict:
+        """REDUCE step: combine chunk-summaries into a strict JSON with
+        markdown_summary, key_points, category, and language.
+
+        Returns a dict (parsed JSON), NOT a raw string.
+        """
         combined_text = " ".join(seg["summary"] for seg in segments_list)
         clean_combined = WHITESPACE_RE.sub(" ", combined_text.strip())[:3000]
 
         if not clean_combined:
-            return f"Summary of: {video_title}."
+            return {
+                "markdown_summary": f"Summary of: {video_title}.",
+                "key_points": [],
+                "category": "Education",
+                "language": "ar",
+            }
+
+        categories_str = ", ".join(f'"{c}"' for c in self.VALID_CATEGORIES)
 
         if INFERENCE_MODE == "groq":
             messages = [
                 {
                     "role": "system",
                     "content": (
-                        "You are a professional summarizer compiling a final overview of a video.\n"
-                        f"{_LANG_MATCH_INSTRUCTION}\n\n"
-                        "Below are partial summaries from different sections of the video. "
-                        "Combine and structure them into a highly cohesive study note. "
-                        "You MUST structure your response EXACTLY as follows:\n\n"
-                        "## 📋 [General Summary / الملخص العام]\n"
-                        "(Write a brief 2-3 sentence overview of the entire video here.)\n\n"
-                        "## ❓ [Key Questions & Answers / أبرز الأسئلة والأجوبة]\n"
-                        "(Extract exactly the 5 most important insights and format them as Q&A. Use exactly: Q: [Question]? \\n A: [Answer] for each turn. Do not use bold/italic formatting on 'Q:' or 'A:' to keep it clean.)\n\n"
-                        "## 💡 [Key Points / أهم النقاط]\n"
-                        "(Write 3 to 5 actual, factual bullet points extracted directly from the text. DO NOT use generic placeholder text like 'الاستنتاج الجوهري' or generic statements. Ensure they are concrete details.)\n\n"
-                        "CRITICAL Rules:\n"
-                        "1. Translate all section headers to match the detected primary language of the video transcript automatically. "
-                        "For example, if the video transcript is in Arabic, all section headers must be in Arabic (e.g. '## 📋 الملخص العام', '## ❓ أبرز الأسئلة والأجوبة', '## 💡 أهم النقاط'). "
-                        "If the video transcript is in English, all section headers must be in English (e.g. '## 📋 General Summary', '## ❓ Key Questions & Answers', '## 💡 Key Points').\n"
-                        "2. DO NOT use placeholder text under any circumstances.\n"
-                        "3. Do not repeat headers unnecessarily. Ensure the output is highly cohesive.\n"
-                        "4. Output ONLY the structured Markdown notes, nothing else."
+                        "You are a professional summarizer compiling a final overview of a video.\n\n"
+                        "STRICT RULES — follow every one exactly:\n\n"
+                        "1. LANGUAGE: Detect the primary language of the text below. "
+                        "If Arabic, write EVERYTHING (headers, body, questions, answers, key points) in Arabic. "
+                        "If English, write EVERYTHING in English. "
+                        "NEVER mix languages.\n\n"
+                        "2. OUTPUT FORMAT: You MUST return a single valid JSON object with this exact schema — no extra text, no markdown fences, ONLY the JSON:\n"
+                        '{\n'
+                        '  "language": "ar" or "en",\n'
+                        '  "category": "one of the categories listed below",\n'
+                        '  "markdown_summary": "the formatted markdown string",\n'
+                        '  "key_points": ["point 1", "point 2", "point 3", "point 4", "point 5"]\n'
+                        '}\n\n'
+                        '3. MARKDOWN_SUMMARY FORMATTING — this is critical for readability:\n'
+                        '   - Start with: ## 📋 الملخص العام  (or ## 📋 General Summary for English)\n'
+                        '   - Then TWO newlines (\\n\\n)\n'
+                        '   - Write a clear, concise 1-paragraph overview of the entire video.\n'
+                        '   - Then: \\n\\n---\\n\\n\n'
+                        '   - Then: ## ❓ أبرز الأسئلة والأجوبة  (or ## ❓ Key Questions & Answers for English)\n'
+                        '   - Then TWO newlines (\\n\\n)\n'
+                        '   - Write exactly 5 Q&A pairs. Format each pair as:\n'
+                        '     **س: [Question]?**\\nج: [Answer]\\n\\n  (for Arabic)\n'
+                        '     **Q: [Question]?**\\nA: [Answer]\\n\\n  (for English)\n'
+                        '   - IMPORTANT: Put \\n\\n (double newline) between EVERY Q&A pair for spacious layout.\n'
+                        '   - Use relevant modern emojis sparingly in questions to make it engaging.\n'
+                        '   - DO NOT include key points, bullet lists, or any other sections in markdown_summary.\n\n'
+                        '4. KEY_POINTS: Exactly 5 concise, factual strings in a JSON array. "
+                        "These must be concrete insights from the content, NOT generic text. "
+                        "Do NOT repeat these inside markdown_summary.\n\n"
+                        f"5. CATEGORY: Must be exactly one of: [{categories_str}]. "
+                        "Choose the most accurate one based on the actual content.\n\n"
+                        "6. Do NOT wrap the output in markdown code fences. Return raw JSON only."
                     ),
                 },
                 {"role": "user", "content": clean_combined},
             ]
-            logger.info("🟢 Reducing summaries via Groq API (llama-3.3-70b-versatile) to Q&A Markdown...")
+            logger.info("🟢 Reducing summaries via Groq API (strict JSON mode)...")
             groq_client = get_groq_client()
             chat_completion = groq_client.chat.completions.create(
                 model="llama-3.3-70b-versatile",
                 messages=messages,
-                max_tokens=1000,
+                max_tokens=1500,
                 temperature=0.0,
+                response_format={"type": "json_object"},
             )
-            overall = chat_completion.choices[0].message.content or ""
+            raw_response = chat_completion.choices[0].message.content or ""
+            logger.info(f"🔎 Reduce raw response (len={len(raw_response)}): {raw_response[:300]!r}")
+            return self._parse_reduce_json(raw_response, video_title)
         else:
+            # Local Qwen mode — keep simple text-based reduce (no JSON)
             messages = [
                 {
                     "role": "system",
@@ -309,10 +367,15 @@ class NoteGenerator:
             logger.info("🤖 Reducing summaries via local Qwen pipeline...")
             overall = _generate_text_local(messages, max_new_tokens=250)
 
-        if not overall or len(overall.strip()) < 5:
-            overall = f"استعراض شامل ومناقشة تفصيلية لموضوع: {video_title}."
+            if not overall or len(overall.strip()) < 5:
+                overall = f"استعراض شامل ومناقشة تفصيلية لموضوع: {video_title}."
 
-        return overall
+            return {
+                "markdown_summary": overall,
+                "key_points": [],
+                "category": "Education",
+                "language": "ar",
+            }
 
     def generateSummary(self, transcript_text: str, video_title: str) -> Dict:
         """Generates a structured AI summary, validated against SummarySchema."""

src/summarization/schemas.py

@@ -73,3 +73,10 @@ class SummarySchema(BaseModel):
             " Examples: ['Python', 'Machine Learning', 'Neural Networks']."
         ),
     )
+    key_points: List[str] = Field(
+        default_factory=list,
+        description=(
+            "Exactly 5 concise key points extracted from the video content."
+            " Used by the UI to render a separate key-points list."
+        ),
+    )