feat: implement local model inference with Qwen2.5 and mDeBERTa, and add recommendation system service.

pyproject.toml

@@ -11,7 +11,7 @@ dependencies = [
     "email-validator>=2.3.0",
     "fastapi==0.109.0",
     "google-api-python-client==2.115.0",
-    "groq>=0.9.0",
+    "accelerate>=0.27.0",
     "greenlet==3.3.1",
     "httpx==0.26.0",
     "openai-whisper==20250625",

requirements.txt

@@ -5,11 +5,18 @@ bgutil-ytdlp-pot-provider==1.3.1
 youtube-transcript-api==0.6.2
 curl_cffi
 
-# --- AI, LLMs & Transcription Fallback ---
-groq>=0.9.0
-openai-whisper==20250625
-torch
+# --- AI & Local Models (CPU-only, HuggingFace Spaces 16GB) ---
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch==2.2.2+cpu
 torchaudio
+transformers==4.38.2
+sentencepiece==0.2.0
+accelerate>=0.27.0
+protobuf==4.25.3
+langdetect>=1.0.9
+
+# --- Whisper (local transcription fallback) ---
+openai-whisper==20250625
 
 # --- Backend Infrastructure (FastAPI) ---
 fastapi==0.109.0
@@ -18,6 +25,7 @@ pydantic==2.12.5
 pydantic[email]
 email-validator
 pydantic-settings==2.1.0
+pydantic-core==2.41.5
 python-multipart==0.0.6
 python-dotenv==1.0.0
 httpx[socks]==0.26.0
@@ -31,28 +39,9 @@ greenlet==3.3.1
 passlib[bcrypt]==1.7.4
 python-jose[cryptography]==3.3.0
 bcrypt==4.1.2
-pydantic-core==2.41.5
 
 # --- Integration & Utilities ---
 google-api-python-client==2.115.0
 firebase-admin==6.5.0
 dnspython
-
-# --- ML & Recommendations ---
-# keybert
-# sentence-transformers
 edge-tts>=6.1.9
-# --- ML & Summeraization ---
-transformers>=4.35.0
-torch>=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu
-sentencepiece>=0.1.99
-protobuf>=3.20.3
-pydantic>=2.0
-langdetect>=1.0.9
---extra-index-url https://download.pytorch.org/whl/cpu
-torch==2.2.2+cpu
-transformers==4.38.2
-sentencepiece==0.2.0
-protobuf==4.25.3
-pydantic>=2.0
-langdetect>=1.0.9

src/api/chat_routes.py

@@ -1,5 +1,5 @@
 """
-Chat routes — Document-specific Q&A powered by Groq.
+Chat routes — Document-specific Q&A powered by local Qwen model.
 """
 
 from typing import List, Optional

src/categorization/topic_classifier.py

@@ -16,6 +16,10 @@ Categories:
 
 from typing import List, Set
 
+from src.utils.logger import setup_logger
+
+logger = setup_logger(__name__)
+
 
 # ─────────────────────────────────────────────────────────────────────────────
 # PREDEFINED CATEGORIES
@@ -257,3 +261,72 @@ def get_primary_category(topics: List[str]) -> str:
     Alias for classify_topic().
     """
     return classify_topic(topics)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# ZERO-SHOT CLASSIFICATION (mDeBERTa fallback)
+# ─────────────────────────────────────────────────────────────────────────────
+
+def classify_topic_zeroshot(text: str) -> str:
+    """Classify free-form text into one of the predefined UI categories
+    using the mDeBERTa zero-shot classification pipeline.
+
+    Args:
+        text: Free-form text (transcript excerpt, note body, etc.)
+
+    Returns:
+        The best-matching category string from CATEGORIES.
+    """
+    if not text or not text.strip():
+        return "Education & Science"
+
+    try:
+        from src.utils.model_loader import get_classifier_pipeline
+
+        classifier = get_classifier_pipeline()
+
+        # Truncate to ~500 chars for speed on CPU
+        result = classifier(
+            text[:500],
+            candidate_labels=CATEGORIES,
+            multi_label=False,
+        )
+
+        best_label = result["labels"][0]
+        best_score = result["scores"][0]
+        logger.info(
+            "🏷️ Zero-shot classification: %s (score=%.3f)", best_label, best_score
+        )
+        return best_label
+
+    except Exception as e:
+        logger.warning("⚠️ Zero-shot classification failed: %s — falling back", e)
+        return "Education & Science"
+
+
+def classify_topic_hybrid(topics: List[str], text: str = "") -> str:
+    """Best-of-both-worlds classifier.
+
+    1. First tries fast keyword matching via ``classify_topic(topics)``.
+    2. If the result is the generic fallback ("Education & Science") AND
+       ``text`` is provided, runs the mDeBERTa zero-shot classifier on
+       the text for a more nuanced result.
+
+    Args:
+        topics: List of topic strings (from the summarization pipeline).
+        text:   Optional free-form text for zero-shot fallback.
+
+    Returns:
+        A single category string from CATEGORIES.
+    """
+    keyword_result = classify_topic(topics)
+
+    # If keyword matching gave a confident answer, use it
+    if keyword_result != "Education & Science":
+        return keyword_result
+
+    # If we have text, try zero-shot as a fallback
+    if text and text.strip():
+        return classify_topic_zeroshot(text)
+
+    return keyword_result

src/recommendation/README.md

@@ -48,7 +48,7 @@ async def test_recommendations():
 
 ## Libraries Used
 - `google-api-python-client` - Search YouTube.
-- `groq` - Extract topics via LLM.
+- `transformers` - Extract topics via local Qwen model.
 
 ## Important Notes
 - YouTube API has a **daily quota limit**, use caching to reduce requests.

src/recommendation/recommender.py

@@ -3,9 +3,8 @@ from collections import Counter
 from typing import List, Dict, Optional
 from googleapiclient.discovery import build
 from src.utils.logger import setup_logger
+from src.utils.model_loader import generate_text
 import random
-# import anthropic
-from groq import Groq
 
 logger = setup_logger(__name__)
 
@@ -17,14 +16,13 @@ class RecommendationService:
     Service for suggesting videos based on user's saved notes.
     Pipeline:
       1. Top 3 most-repeated categories across all user notes
-      2. Extract key keywords from the latest note per category (via Claude)
+      2. Extract key keywords from the latest note per category (via local Qwen model)
       3. Build a YouTube search query and return recommendations
     """
 
     def __init__(self, api_key: Optional[str] = None):
         self.api_key = "AIzaSyA3erB-Lxd5SOoBOXaumOCVaEr3TcgYG60"
         self.youtube = build("youtube", "v3", developerKey=self.api_key)
-        self.groq_client = Groq(api_key="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") 
 
     # ──────────────────────────────────────────────
     # Step 1: top 3 categories
@@ -72,7 +70,7 @@ class RecommendationService:
         }
     
     # ──────────────────────────────────────────────
-    # Step 3: extract keywords via grok(llama-3.3-70b-versatile)
+    # Step 3: extract keywords via local Qwen model
     # ──────────────────────────────────────────────
     async def _extract_keywords_with_claude(
         self, notes: List[Dict], category: str  
@@ -98,16 +96,15 @@ class RecommendationService:
 
         try:
             loop = asyncio.get_event_loop()
-            # groq_client = Groq(api_key="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")
-            response = await loop.run_in_executor(
+            # Run local model inference in a thread pool to avoid blocking the event loop
+            raw = await loop.run_in_executor(
                 None,
-                lambda: self.groq_client.chat.completions.create(
-                    model="llama-3.3-70b-versatile",
-                    messages=[{"role": "user", "content": prompt}],
-                    max_tokens=120,
-                )
+                lambda: generate_text(
+                    [{"role": "user", "content": prompt}],
+                    max_new_tokens=80,
+                ),
             )
-            raw = response.choices[0].message.content.strip()
+
             import json, re
             # strip accidental markdown fences
             raw = re.sub(r"```json|```", "", raw).strip()
@@ -116,7 +113,7 @@ class RecommendationService:
                 logger.info(f" Keywords for '{category}': {keywords}")
                 return [str(k) for k in keywords[:5]]
         except Exception as e:
-            logger.warning(f" Grok keyword extraction failed for '{category}': {e}")
+            logger.warning(f" Local keyword extraction failed for '{category}': {e}")
 
         return [category]  # fallback
 
@@ -220,7 +217,7 @@ class RecommendationService:
             logger.info(" No valid categories — falling back")
             return await self.get_youtube_recommendations("educational tutorials", limit)
 
-        # ── Step 2: keywords via Claude ──────────
+        # ── Step 2: keywords via local Qwen model ──────────
         latest_notes = self._latest_notes_per_category(notes, top_categories, top_n=2)
 
         valid_categories = [

src/summarization/note_generator.py

@@ -2,11 +2,10 @@ import os
 import re
 from typing import Dict, List
 
-import torch
-from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 from pydantic import ValidationError
 
 from ..utils.logger import setup_logger
+from ..utils.model_loader import generate_text
 from .segmenter import TranscriptSegmenter
 from .schemas import SummarySchema, SegmentSchema
 
@@ -27,7 +26,7 @@ WHITESPACE_RE = re.compile(r"\s+")
 
 
 class NoteGenerator:
-    """Real AI Summarization Engine using a lightweight multilingual mT5 model.
+    """AI Summarization Engine using Qwen2.5-1.5B-Instruct (local, CPU).
 
     Pipeline (map-reduce):
       1. MAP    -> split transcript into word-bounded chunks (TranscriptSegmenter)
@@ -40,32 +39,18 @@ class NoteGenerator:
     """
 
     def __init__(self, chunk_size_words: int = 350):
-        self.model_id = os.environ.get("MT5_MODEL_NAME", "csebuetnlp/mT5_multilingual_XLSum")
-        logger.info(f"🤖 Initializing real local AI Model: {self.model_id}...")
-
-        cache_dir = os.path.join(os.getcwd(), "hf_cache")
-        os.makedirs(cache_dir, exist_ok=True)
-
-        self.tokenizer = AutoTokenizer.from_pretrained(
-            self.model_id, use_fast=False, cache_dir=cache_dir
-        )
-        self.device = "cuda" if torch.cuda.is_available() else "cpu"
-        self.model = AutoModelForSeq2SeqLM.from_pretrained(
-            self.model_id, cache_dir=cache_dir
-        ).to(self.device)
-
         self.segmenter = TranscriptSegmenter(max_segment_words=chunk_size_words)
         self.chunk_size_words = chunk_size_words
+        logger.info("📝 NoteGenerator ready (Qwen2.5-1.5B-Instruct, CPU).")
 
-        logger.info("✅ Real AI Model loaded successfully and ready!")
+    # ── Core summarization via local Qwen model ──────────────────────────
 
     def _run_model_summarization(
         self,
         text: str,
-        max_length: int = 84,
-        num_beams: int = 2,
+        max_new_tokens: int = 200,
     ) -> str:
-        """Run the local mT5 model on a single piece of text and return its summary."""
+        """Run the local Qwen model on a single piece of text and return its summary."""
         try:
             clean_text = WHITESPACE_RE.sub(" ", text.strip())
             if not clean_text:
@@ -74,27 +59,22 @@ class NoteGenerator:
 
             logger.info(f"🔎 Input to model (len={len(clean_text)} chars): {clean_text[:120]!r}...")
 
-            inputs = self.tokenizer(
-                [clean_text],
-                truncation=True,
-                padding="longest",
-                max_length=512,
-                return_tensors="pt",
-            ).to(self.device)
-
-            logger.info(f"🔎 Tokenized input_ids shape: {inputs['input_ids'].shape}")
-
-            with torch.no_grad():
-                summary_ids = self.model.generate(
-                    inputs["input_ids"],
-                    num_beams=num_beams,
-                    max_length=max_length,
-                    no_repeat_ngram_size=2,
-                    early_stopping=True,
-                )
+            messages = [
+                {
+                    "role": "system",
+                    "content": (
+                        "You are a professional summarizer. "
+                        "Summarize the following text concisely in the SAME language as the input. "
+                        "Focus on the main ideas and key points. "
+                        "Output ONLY the summary, nothing else."
+                    ),
+                },
+                {"role": "user", "content": clean_text[:3000]},
+            ]
 
-            result = self.tokenizer.decode(summary_ids[0], skip_special_tokens=True)
-            logger.info(f"🔎 Raw model output (len={len(result)} chars): {result!r}")
+            result = generate_text(messages, max_new_tokens=max_new_tokens)
+
+            logger.info(f"🔎 Model output (len={len(result)} chars): {result[:200]!r}")
 
             if not result or len(result.strip()) < 5:
                 logger.warning(
@@ -174,7 +154,7 @@ class NoteGenerator:
     def _reduce_summary(self, segments_list: List[Dict], video_title: str) -> str:
         """REDUCE step: summarize the combined chunk-summaries into one overall summary."""
         combined_text = " ".join(seg["summary"] for seg in segments_list)
-        overall = self._run_model_summarization(combined_text, max_length=130, num_beams=4)
+        overall = self._run_model_summarization(combined_text, max_new_tokens=250)
 
         if not overall or len(overall.strip()) < 5:
             overall = f"استعراض شامل ومناقشة تفصيلية لموضوع: {video_title}."
@@ -183,7 +163,7 @@ class NoteGenerator:
 
     def generateSummary(self, transcript_text: str, video_title: str) -> Dict:
         """Generates a structured AI summary, validated against SummarySchema."""
-        logger.info("📝 Real AI summary generation triggered (map-reduce pipeline).")
+        logger.info("📝 AI summary generation triggered (map-reduce pipeline, Qwen local).")
         logger.info(
             f"🔎 Received video_title={video_title!r}, "
             f"transcript_text length={len(transcript_text) if transcript_text else 0} chars, "
@@ -211,8 +191,8 @@ class NoteGenerator:
             "summary": overall_summary,
             "segments": segments_list,
             "suggested_category": "General",
-            "conclusion": f"تم التلخيص بنجاح باستخدام موديل mT5 المطور لـ {video_title}.",
-            "topics": ["تلخيص تلقائي", "ذكاء اصطناعي محلي", "mT5-XLSum"],
+            "conclusion": f"تم التلخيص بنجاح باستخدام نموذج Qwen المحلي لـ {video_title}.",
+            "topics": ["تلخيص تلقائي", "ذكاء اصطناعي محلي", "Qwen2.5"],
         }
 
         # 3. VALIDATE — enforce the schema contract before returning
@@ -238,8 +218,8 @@ class NoteGenerator:
                 )
                 for i in range(3)
             ],
-            conclusion=f"تم التلخيص بنجاح باستخدام موديل mT5 المطور لـ {video_title}.",
-            topics=["تلخيص تلقائي", "ذكاء اصطناعي محلي", "mT5-XLSum"],
+            conclusion=f"تم التلخيص بنجاح باستخدام نموذج Qwen المحلي لـ {video_title}.",
+            topics=["تلخيص تلقائي", "ذكاء اصطناعي محلي", "Qwen2.5"],
         )
         return fallback.model_dump()
 
@@ -301,5 +281,42 @@ class NoteGenerator:
     def chat_with_note(
         self, note_content: str, question: str, history: list[dict] | None = None
     ) -> str:
-        """Instant stable QA response."""
-        return "تم تفعيل الموديل المحلي بنجاح! نظام الأسئلة والأجوبة سيتم ربطه في التحديث القادم."
+        """Answer a question about a note using the local Qwen model.
+
+        The model is instructed to ground its answers solely in the provided
+        note content to avoid hallucination.
+        """
+        try:
+            messages = [
+                {
+                    "role": "system",
+                    "content": (
+                        "You are a helpful study assistant. "
+                        "Answer the user's question based ONLY on the note content provided below. "
+                        "If the answer is not in the note, say so honestly. "
+                        "Reply in the same language the user uses.\n\n"
+                        f"--- NOTE CONTENT ---\n{note_content[:4000]}\n--- END NOTE ---"
+                    ),
+                },
+            ]
+
+            # Include conversation history if available
+            if history:
+                for msg in history[-6:]:  # Keep last 6 turns to fit context
+                    role = msg.get("role", "user")
+                    content = msg.get("content", "")
+                    if role in ("user", "assistant") and content:
+                        messages.append({"role": role, "content": content})
+
+            messages.append({"role": "user", "content": question})
+
+            answer = generate_text(messages, max_new_tokens=300)
+
+            if not answer or len(answer.strip()) < 3:
+                return "عذرًا، لم أتمكن من توليد إجابة. يرجى إعادة صياغة السؤال."
+
+            return answer
+
+        except Exception as e:
+            logger.error(f"❌ Chat error: {e}", exc_info=True)
+            return "حدث خطأ أثناء معالجة سؤالك. يرجى المحاولة مرة أخرى."

src/transcription/downloader.py

@@ -20,7 +20,7 @@ class NoTranscriptError(RuntimeError):
 # ─────────────────────────────────────────────────────────────────────────────
 
 class YouTubeDownloader:
-    """Extracts YouTube transcripts via Supadata or Deep Scan (Groq Whisper)."""
+    """Extracts YouTube transcripts via Supadata or Deep Scan (local Whisper)."""
 
     def __init__(self):
         self._supadata_key = os.environ.get("SUPADATA_API_KEY", "").strip()

src/utils/model_loader.py

@@ -0,0 +1,162 @@
+"""
+Lazy-loading singleton for local AI models.
+
+Manages two models for CPU inference on HuggingFace Spaces (16 GB RAM):
+
+1. **Qwen2.5-1.5B-Instruct**  – causal LM for summarization, chat Q&A,
+   and keyword extraction (replaces all former Groq calls).
+2. **mDeBERTa-v3-base-xnli-multiset** – zero-shot classifier for topic
+   categorization.
+
+Both models are loaded *lazily* on first use and cached for the lifetime
+of the process, keeping cold-start memory low.
+"""
+
+import os
+import threading
+from typing import Tuple
+
+import torch
+from transformers import (
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    pipeline as hf_pipeline,
+)
+
+from src.utils.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+# ── Configuration ────────────────────────────────────────────────────────────
+QWEN_MODEL_ID = os.environ.get(
+    "QWEN_MODEL_ID", "Qwen/Qwen2.5-1.5B-Instruct"
+)
+CLASSIFIER_MODEL_ID = os.environ.get(
+    "CLASSIFIER_MODEL_ID", "MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7"
+)
+HF_CACHE_DIR = os.path.join(os.getcwd(), "hf_cache")
+os.makedirs(HF_CACHE_DIR, exist_ok=True)
+
+# ── Internal state (module-level singletons) ─────────────────────────────────
+_qwen_lock = threading.Lock()
+_clf_lock = threading.Lock()
+
+_qwen_model = None
+_qwen_tokenizer = None
+_classifier_pipe = None
+
+
+# ── Public API ───────────────────────────────────────────────────────────────
+
+def get_qwen_model() -> Tuple:
+    """Return ``(model, tokenizer)`` for Qwen2.5-1.5B-Instruct.
+
+    Loads on first call; subsequent calls return the cached objects.
+    """
+    global _qwen_model, _qwen_tokenizer
+
+    if _qwen_model is not None:
+        return _qwen_model, _qwen_tokenizer
+
+    with _qwen_lock:
+        # Double-check after acquiring the lock
+        if _qwen_model is not None:
+            return _qwen_model, _qwen_tokenizer
+
+        logger.info("🤖 Loading Qwen model: %s (CPU, float32) …", QWEN_MODEL_ID)
+
+        _qwen_tokenizer = AutoTokenizer.from_pretrained(
+            QWEN_MODEL_ID,
+            cache_dir=HF_CACHE_DIR,
+            trust_remote_code=True,
+        )
+        _qwen_model = AutoModelForCausalLM.from_pretrained(
+            QWEN_MODEL_ID,
+            cache_dir=HF_CACHE_DIR,
+            torch_dtype=torch.float32,
+            device_map="cpu",
+            trust_remote_code=True,
+        )
+        _qwen_model.eval()
+
+        logger.info("✅ Qwen model loaded successfully.")
+        return _qwen_model, _qwen_tokenizer
+
+
+def get_classifier_pipeline():
+    """Return a zero-shot-classification ``Pipeline`` backed by mDeBERTa.
+
+    Loads on first call; subsequent calls return the cached pipeline.
+    """
+    global _classifier_pipe
+
+    if _classifier_pipe is not None:
+        return _classifier_pipe
+
+    with _clf_lock:
+        if _classifier_pipe is not None:
+            return _classifier_pipe
+
+        logger.info(
+            "🤖 Loading zero-shot classifier: %s (CPU) …", CLASSIFIER_MODEL_ID
+        )
+
+        _classifier_pipe = hf_pipeline(
+            "zero-shot-classification",
+            model=CLASSIFIER_MODEL_ID,
+            device=-1,  # CPU
+            cache_dir=HF_CACHE_DIR,
+        )
+
+        logger.info("✅ Zero-shot classifier loaded successfully.")
+        return _classifier_pipe
+
+
+def generate_text(
+    prompt_messages: list[dict],
+    *,
+    max_new_tokens: int = 200,
+    temperature: float = 1.0,
+    do_sample: bool = False,
+) -> str:
+    """High-level helper: run Qwen chat completion and return the reply text.
+
+    Parameters
+    ----------
+    prompt_messages:
+        List of ``{"role": ..., "content": ...}`` dicts compatible with
+        ``tokenizer.apply_chat_template``.
+    max_new_tokens:
+        Cap on generated tokens (keep low for CPU speed).
+    temperature / do_sample:
+        Sampling config.  Defaults to greedy (deterministic).
+    """
+    model, tokenizer = get_qwen_model()
+
+    input_text = tokenizer.apply_chat_template(
+        prompt_messages,
+        tokenize=False,
+        add_generation_prompt=True,
+    )
+
+    inputs = tokenizer(
+        input_text,
+        return_tensors="pt",
+        truncation=True,
+        max_length=2048,
+    ).to("cpu")
+
+    prompt_len = inputs["input_ids"].shape[1]
+
+    with torch.no_grad():
+        output_ids = model.generate(
+            **inputs,
+            max_new_tokens=max_new_tokens,
+            do_sample=do_sample,
+            temperature=temperature if do_sample else 1.0,
+            pad_token_id=tokenizer.eos_token_id,
+        )
+
+    # Decode only the newly generated tokens
+    new_tokens = output_ids[0][prompt_len:]
+    return tokenizer.decode(new_tokens, skip_special_tokens=True).strip()