Upload folder using huggingface_hub

.dockerignore

@@ -0,0 +1,37 @@
+# Environment & secrets
+.env
+.env.*
+
+# Python cache
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+
+# Git
+.git/
+.gitignore
+
+# Tests & local tooling
+tests/
+.pytest_cache/
+
+# Local experiment tracking
+mlflow.db
+mlruns/
+
+# Dev & notebooks
+*.ipynb
+
+# OS files
+.DS_Store
+
+# Build artifacts
+build/
+dist/
+*.egg-info/
+
+# Not needed on HuggingFace
+script.py
+src/training/
+src/ingestion/

.gitattributes

@@ -1,35 +1,36 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.index filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,56 @@
+# 1. البيئة والمتغيرات
+venv/
+env/
+.venv/
+.env
+.env.local
+
+# 2. بايثون
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.python-version
+
+# 3. FAISS والـ Embedding
+# *.faiss
+# *.index
+# faiss_index/
+
+# 4. MLflow (كله لا ترفعه)
+mlflow.db
+mlruns/
+.mlflow/
+metadata.json  # إذا كان تابع لـ MLflow
+
+# 5. الموديلات
+models/
+*.pkl
+*.joblib
+*.h5
+*.keras
+*.pt
+*.bin
+
+# 6. الاختبارات
+.pytest_cache/
+.coverage
+htmlcov/
+.coverage.*
+
+# 7. ملفات مؤقتة
+temp/
+tmp/
+uploads/
+*.log
+logs/
+
+# 8. أنظمة التشغيل
+.DS_Store
+Thumbs.db
+
+# 9. الـ IDEs
+.vscode/
+.idea/
+*.swp
+*.swo

Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+RUN python - <<EOF
+from sentence_transformers import SentenceTransformer
+SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")
+EOF
+
+COPY . .
+
+ENV PYTHONPATH=/app/src
+
+EXPOSE 7860
+
+CMD ["uvicorn", "src.api.main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,11 +1,12 @@
----
-title: Rag Lecture Saver Api
-emoji: 👀
-colorFrom: green
-colorTo: green
-sdk: docker
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Rag Lecture Saver
+emoji: 🏃
+colorFrom: green
+colorTo: yellow
+sdk: docker
+pinned: false
+license: mit
+app_port: 7860
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,64 @@
+import sys
+from pathlib import Path
+
+ROOT_DIR = Path(__file__).resolve().parent.parent
+if str(ROOT_DIR) not in sys.path:
+    sys.path.insert(0, str(ROOT_DIR))
+
+import streamlit as st
+import requests
+
+API_URL = "http://0.0.0.0:8000/ask"
+
+st.set_page_config(
+    page_title="Lecture RAG",
+    layout="wide",
+)
+
+st.title("📚 Lecture RAG Assistant")
+
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+
+chat_container = st.container()
+
+with chat_container:
+    for msg in st.session_state.messages:
+        with st.chat_message(msg["role"]):
+            st.markdown(msg["content"])
+
+question = st.chat_input("اكتب سؤالك هنا / Ask your question")
+
+if question:
+    st.session_state.messages.append({"role": "user", "content": question})
+    with chat_container:
+        with st.chat_message("user"):
+            st.markdown(question)
+
+        with st.chat_message("assistant"):
+            with st.spinner("Thinking..."):
+                try:
+                    response = requests.post(
+                        API_URL,
+                        json={"question": question},
+                        timeout=120
+                    )
+                    response.raise_for_status()
+                    data = response.json()
+                    answer = data.get("answer", "No answer returned.")
+                except Exception as e:
+                    data = {}
+                    answer = f"❌ API error: {e}"
+
+                st.markdown(answer)
+
+                if data.get("citations"):
+                    with st.expander("📌 Sources"):
+                        seen = set()
+                        for c in data["citations"].values():
+                            key = (c["source"], c["page"])
+                            if key not in seen:
+                                seen.add(key)
+                                st.markdown(f"- 📄 `{c['source']}` | page `{c['page']}`")
+
+    st.session_state.messages.append({"role": "assistant", "content": answer})

faiss.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfff733e02cc164e9ca95eedb12cd27368f78de9014d6a359985dc60f5b9a477
+size 222765

requirements.txt

@@ -0,0 +1,8 @@
+fastapi
+uvicorn
+pydantic
+python-multipart
+sentence-transformers
+faiss-cpu
+groq
+langchain-text-splitters

src/api/main.py

@@ -0,0 +1,89 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from pathlib import Path
+import json
+import faiss
+import logging
+from contextlib import asynccontextmanager
+
+from src.embedding.embedding_client import EmbeddingClient
+import os
+from src.embedding.embeddings import EmbeddingModel
+from src.vector_store.FAISS_store import VectorStore
+from src.services.rag_service import RAGService
+
+logger = logging.getLogger("rag-api")
+logging.basicConfig(level=logging.INFO)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    try:
+        logger.info("Starting RAG API...")
+
+        faiss_path = Path("faiss.index")
+        metadata_path = Path("metadata.json")
+
+        if not faiss_path.exists() or not metadata_path.exists():
+            logger.warning("FAISS or metadata not found. API will start in degraded mode.")
+            app.state.rag_service = None
+            yield
+            return
+
+        embedder = EmbeddingModel()
+        index = faiss.read_index(str(faiss_path))
+        store = VectorStore(dim=index.d)
+        store.index = index
+
+        with open(metadata_path, "r", encoding="utf-8") as f:
+            store.metadata = json.load(f)
+
+        app.state.store = store
+        app.state.rag_service = RAGService(store, embedder)
+
+        logger.info("RAG API ready.")
+        yield
+
+    except Exception as e:
+        logger.exception("Startup failed")
+        raise e
+
+
+app = FastAPI(
+    title="Lecture-Saver 3000 API",
+    version="1.0.0",
+    lifespan=lifespan,
+)
+
+
+class ChatRequest(BaseModel):
+    question: str
+
+
+@app.get("/health")
+def health():
+    return {
+        "status": "ok",
+        "rag_loaded": app.state.rag_service is not None
+    }
+
+
+@app.post("/ask")
+async def ask(request: ChatRequest):
+    if app.state.rag_service is None:
+        raise HTTPException(
+            status_code=503,
+            detail="RAG service not initialized"
+        )
+
+    try:
+        response = await app.state.rag_service.answer_question(request.question)
+    except Exception as e:
+        logger.exception("Unhandled RAG error")
+        raise HTTPException(status_code=500, detail="Internal RAG failure")
+
+    if response.get("status") == "error":
+        status_code = 503 if response.get("error_type") == "LLMGenerationError" else 500
+        raise HTTPException(status_code=status_code, detail=response.get("message"))
+
+    return response

src/core/exceptions.py

@@ -0,0 +1,10 @@
+
+class RAGException(Exception):
+    
+    pass
+
+class RetrievalError(RAGException):
+    pass
+
+class LLMGenerationError(RAGException):
+    pass

src/embedding/embedding_client.py

@@ -0,0 +1,23 @@
+import httpx
+from typing import List
+import logging
+
+logger = logging.getLogger("embedding-client")
+
+class EmbeddingClient:
+    def __init__(self, base_url: str):
+        self.base_url = base_url.rstrip("/")
+
+    async def encode(self, texts: List[str]) -> List[List[float]]:
+        try:
+            async with httpx.AsyncClient(timeout=30) as client:
+                response = await client.post(
+                    f"{self.base_url}/embed",
+                    json={"texts": texts}
+                )
+                response.raise_for_status()
+                return response.json()["embeddings"]
+
+        except Exception as e:
+            logger.exception("Embedding service call failed")
+            raise RuntimeError("EmbeddingServiceError") from e

src/embedding/embeddings.py

@@ -0,0 +1,18 @@
+from sentence_transformers import SentenceTransformer
+import numpy as np
+from typing import List
+
+class EmbeddingModel:
+    def __init__(self , model_name:str = "paraphrase-multilingual-MiniLM-L12-v2"):
+        self.model_name = model_name
+        self.model = SentenceTransformer(
+            model_name)
+
+    def encode(self , texts: List[str])-> np.ndarray:
+        
+        return self.model.encode(
+            texts,
+            show_progress_bar=False,
+            convert_to_numpy=True,
+            normalize_embeddings=True
+        )

src/evaluation/faithfulness.py

@@ -0,0 +1,21 @@
+from src.generation.llm_client import LLMClient
+
+llm = LLMClient()
+
+FAITHFULNESS_PROMPT = """
+Given the following CONTEXT and ANSWER:
+
+Determine whether the ANSWER is fully supported by the CONTEXT.
+Reply ONLY with one word:
+- YES
+- NO
+"""
+
+def faithfulness_score(context: str, answer: str)-> int:
+
+    response = llm.generate(
+        question=FAITHFULNESS_PROMPT,
+        context_chunks=[f"CONTEXT:\n{context}\n\nANSWER:\n{answer}"]
+    )
+    
+    return 1 if response.strip().upper().startswith("YES") else 0 

src/evaluation/relevance.py

@@ -0,0 +1,9 @@
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+
+def context_relevance(query_embedding , chunk_embedding)-> float:
+
+    
+    sims = cosine_similarity(query_embedding , chunk_embedding)
+
+    return float(np.mean(sims))

src/generation/llm_client.py

@@ -0,0 +1,66 @@
+import os
+import logging
+from typing import List
+from groq import Groq
+import asyncio
+
+logger = logging.getLogger("llm-client")
+
+
+SYSTEM_PROMPT = """
+You are a university teaching assistant.
+
+Rules:
+- Answer the question using ONLY the provided context.
+- You may rephrase or translate the information to match the language of the user's question.
+- Do NOT add new information.
+- Do NOT use external knowledge.
+- When you state a fact, add the reference number in square brackets, e.g. [1], [2].
+- If the answer is not explicitly present, say:
+  "Not found in the provided lecture material."
+"""
+
+
+class LLMClient:
+    def __init__(self):
+        api_key = os.getenv("GROQ_API_KEY")
+        if not api_key:
+            raise RuntimeError("GROQ_API_KEY not set")
+
+        self.model = os.getenv(
+            "GROQ_MODEL",
+            "llama-3.3-70b-versatile"
+        )
+
+        self.client = Groq(api_key=api_key)
+
+    async def generate(
+        self,
+        question: str,
+        context_chunks: List[str],
+        chat_history: List[str] | None = None
+    ) -> str:
+        context = "\n\n".join(context_chunks)
+
+        messages = [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {
+                "role": "user",
+                "content": f"Context:\n{context}\n\nQuestion:\n{question}"
+            }
+        ]
+
+        try:
+            response =  await asyncio.to_thread( 
+                self.client.chat.completions.create,
+                model=self.model,
+                messages=messages,
+                temperature=0.3,
+                max_tokens=512
+                
+            )
+            return response.choices[0].message.content.strip()
+
+        except Exception as e:
+            logger.exception("Groq LLM call failed")
+            raise RuntimeError("LLMGenerationError") from e

src/ingestion/document_loader.py

@@ -0,0 +1,34 @@
+import json
+from pathlib import Path
+from typing import List , Dict
+from src.ingestion.pdf_to_markdown import MarkdownLoader
+from pathlib import Path
+
+def load_documents(path: Path):
+
+    documents: List[Dict] = []
+    loader = MarkdownLoader()
+
+    if path.is_file():
+        if path.suffix.lower() == ".pdf" :
+
+            documents.extend(loader.load_pdf(path))
+
+        else :
+            raise ValueError(f"Unsupported file type: {path.suffix}")
+        
+    elif path.is_dir():
+        pdf_files = sorted(path.rglob("*.pdf"))
+
+        if not pdf_files:
+            raise ValueError("No PDF files found in the directory")
+        
+        for pdf_path in pdf_files :
+            documents.extend(loader.load_pdf(pdf_path))
+
+    else:
+        raise ValueError(f"Invalid path: {path}")
+    
+    return documents
+
+    

src/ingestion/pdf_to_markdown.py

@@ -0,0 +1,91 @@
+import re
+import logging
+from pathlib import Path
+from typing import List, Dict
+from docling.document_converter import DocumentConverter
+from docling_core.types.doc import DocItemLabel, TableItem, TextItem
+from  src.preprocessing.cleaner import normalize_text
+from docling.datamodel.pipeline_options import PdfPipelineOptions
+from docling.document_converter import DocumentConverter, PdfFormatOption
+
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class MarkdownLoader:
+    def __init__(self):
+        options = PdfPipelineOptions()
+        options.do_ocr = False           
+        options.do_table_structure = True 
+
+        self.converter = DocumentConverter(
+            format_options={
+                "pdf": PdfFormatOption(pipeline_options=options)
+            }
+        )
+
+
+    def get_element_markdown(self, element) -> str:
+
+        if isinstance(element, TableItem):
+            try:
+                return element.export_to_markdown()
+            except:
+                return element.text if hasattr(element, 'text') else ""
+        
+        elif isinstance(element, TextItem):
+            text = element.text
+            label = element.label
+            if label == DocItemLabel.TITLE:
+                return f"# {text}"
+            elif label == DocItemLabel.SECTION_HEADER:
+                           
+                return f"## {text}"
+            elif label == DocItemLabel.LIST_ITEM:
+                return f"* {text}"
+            return text
+        
+        return ""
+
+    def load_pdf(self, pdf_path: Path) -> List[Dict]:
+        documents = []
+        try:
+            logger.info(f"Processing.... {pdf_path.name}")
+            result = self.converter.convert(str(pdf_path))
+            doc = result.document
+            
+            pages_content = {}
+
+            for element, level in doc.iterate_items():
+                if hasattr(element, 'prov') and element.prov:
+                    page_no = element.prov[0].page_no
+                    
+                    element_md = self.get_element_markdown(element)
+                    
+                    if element_md:
+                        if page_no not in pages_content:
+                            pages_content[page_no] = []
+                        pages_content[page_no].append(element_md)
+
+            for page_number in sorted(pages_content.keys()):
+                raw_text = "\n\n".join(pages_content[page_number])
+                
+                final_text = normalize_text(raw_text)
+                
+                if final_text.strip():
+                    documents.append({
+                        "text": final_text,
+                        "metadata": {
+                            "source": pdf_path.name,
+                            "page": page_number,
+                            "format": "markdown"
+                        }
+                    })
+            
+            logger.info(f" {len(documents)} successfully")
+            return documents
+
+        except Exception as e:
+            logger.error(f"error in processing {pdf_path.name}: {e}")
+            import traceback
+            logger.error(traceback.format_exc())

src/mlops/tracking.py

@@ -0,0 +1,42 @@
+import mlflow
+from pathlib import Path
+import faiss
+from contextlib import contextmanager
+
+
+def start_experiment(experiment_name: str):
+    mlflow.set_experiment(experiment_name)
+    mlflow.start_run()
+
+
+
+def init_mlflow(experiment_name: str):
+    mlflow.set_experiment(experiment_name)
+
+@contextmanager
+def start_rag_run(run_name: str = "rag_query"):
+    with mlflow.start_run(run_name=run_name):
+        yield
+
+def log_params_dict(params: dict):
+    for k, v in params.items():
+        mlflow.log_param(k, v)
+
+def log_metrics_dict(metrics: dict):
+    for k, v in metrics.items():
+        mlflow.log_metric(k, v)
+
+def log_text_artifact(text: str, artifact_path: str):
+    mlflow.log_text(text, artifact_path)
+
+def log_faiss_index(index, path: Path):
+    faiss.write_index(index, str(path))
+    mlflow.log_artifact(str(path))
+
+def log_artifact_metadatas(metadatas: Path ="metadata.json"): 
+    mlflow.log_artifact(str(metadatas))
+
+
+def end_run():
+    mlflow.end_run()
+

src/preprocessing/chunker_markdown.py

@@ -0,0 +1,143 @@
+from typing import List, Dict
+import re
+
+from langchain_text_splitters import (
+    MarkdownHeaderTextSplitter,
+    RecursiveCharacterTextSplitter,
+)
+
+# --------------------------------------
+# Settings
+# --------------------------------------
+HEADERS_TO_SPLIT_ON = [
+    ("#", "h1"),
+    ("##", "h2"),
+    ("###", "h3"),
+]
+
+SEPARATORS = ["\n\n", "\n", " ", ""]
+
+MIN_CHUNK_LENGTH = 40  
+
+
+# --------------------------------------
+# Main API 
+# --------------------------------------
+def chunk_document(
+    document: Dict,
+    chunk_size: int = 800,
+    overlap: int = 100,
+) -> List[str]:
+    """
+    Final structure-aware chunking for Markdown (Docling output)
+
+    Input:
+        document = {
+            "text": "... markdown ...",
+            "metadata": {
+                "source": "...",
+                "page": int,
+                "format": "markdown"
+            }
+        }
+
+    Output:
+        List[str]  
+    """
+
+    text = document
+
+    # 1) Header-based splitting
+    header_splitter = MarkdownHeaderTextSplitter(
+        headers_to_split_on=HEADERS_TO_SPLIT_ON,
+        strip_headers=False,
+    )
+
+    header_sections = header_splitter.split_text(text)
+
+    # 2) Recursive splitter (size-based)
+    recursive_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=overlap,
+        separators=SEPARATORS,
+    )
+
+    final_chunks: List[str] = []
+
+    for section in header_sections:
+        section_text = section.page_content.strip()
+
+        # -----------------------------
+        # Noise filtering 
+        # -----------------------------
+        if _is_noise(section_text):
+            continue
+
+        # -----------------------------
+        # Table handling
+        # -----------------------------
+        if _looks_like_markdown_table(section_text):
+            final_chunks.append(section_text)
+            continue
+
+        # -----------------------------
+        # Merge header-only chunks
+        # -----------------------------
+        if _is_header_only(section_text):
+            continue
+
+        # -----------------------------
+        # Size-based splitting
+        # -----------------------------
+        sub_chunks = recursive_splitter.split_text(section_text)
+
+        for sub in sub_chunks:
+            sub = sub.strip()
+            if len(sub) < MIN_CHUNK_LENGTH:
+                continue
+
+            final_chunks.append(sub)
+
+    return final_chunks
+
+
+# --------------------------------------
+# Helpers
+# --------------------------------------
+def _looks_like_markdown_table(text: str) -> bool:
+    lines = text.splitlines()
+    if len(lines) < 2:
+        return False
+
+    has_pipes = any("|" in line for line in lines)
+    has_separator = any(
+        re.match(r"^\s*\|?[\s:-]+\|", line) for line in lines
+    )
+
+    return has_pipes and has_separator
+
+
+def _is_header_only(text: str) -> bool:
+    """
+    Detect chunks that are only headers (e.g. '## العنوان')
+    """
+    lines = text.splitlines()
+    if len(lines) != 1:
+        return False
+
+    return lines[0].lstrip().startswith("#")
+
+
+def _is_noise(text: str) -> bool:
+    """
+    Remove garbage chunks: symbols, single letters, etc.
+    """
+    stripped = text.strip()
+
+    if len(stripped) < 10:
+        return True
+
+    if re.fullmatch(r"[■S\s]+", stripped):
+        return True
+
+    return False

src/preprocessing/cleaner.py

@@ -0,0 +1,41 @@
+import re
+
+# --------------------------------------
+# Arabic-safe normalization (RAG oriented)
+# --------------------------------------
+
+ARABIC_DIACRITICS = re.compile(r"[ًٌٍَُِّْـ]")
+
+def normalize_arabic_text(text: str) -> str:
+    """
+    Safe normalization for Arabic PDFs:
+    - Remove diacritics
+    - Normalize punctuation
+    - Preserve word boundaries
+    - DO NOT hallucinate spaces
+    """
+
+    # 1. Remove diacritics
+    text = re.sub(ARABIC_DIACRITICS, "", text)
+
+    # 2. Normalize Arabic punctuation spacing
+    text = re.sub(r"\s*([،؛؟!])\s*", r"\1 ", text)
+    text = re.sub(r"\s*([.:])\s*", r"\1 ", text)
+
+    # 3. Fix obvious header glue (##عنوان)
+    text = re.sub(r"(#+)([^\s#])", r"\1 \2", text)
+
+    # 4. Separate bullets safely
+    text = re.sub(r"\*\s*", "* ", text)
+
+    # 5. Collapse excessive whitespace
+    text = re.sub(r"[ \t]{2,}", " ", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+
+    return text.strip()
+
+
+def normalize_text(text: str) -> str:
+    text = normalize_arabic_text(text)
+    lines = [line.strip() for line in text.splitlines()]
+    return "\n".join(lines)

src/services/rag_service.py

@@ -0,0 +1,85 @@
+from src.embedding.embedding_client import EmbeddingClient
+from src.generation.llm_client import LLMClient
+import asyncio
+import logging
+
+logger = logging.getLogger("rag-service")
+
+class RAGService:
+    def __init__(self, store, embedder :EmbeddingClient):
+        self.store = store
+        self.embedder = embedder
+        self.llm = LLMClient()
+
+    async def answer_question(self, question: str, k: int = 5):
+        
+            if not question or len(question.strip()) == 0:
+                 return {
+                      "status": "error",
+                      "message" : "Empty question"
+                 }
+            
+            # Basic safety limit
+            if len(question) > 2000:
+                 return {
+                      "status" : "error",
+                      "message" : "Question too long"
+                 }
+
+            try:
+
+                # --- Retrieval ---
+                query_vector = self.embedder.encode([question])
+                retrieved_docs = self.store.search(query_vector, k=k)
+
+                if not retrieved_docs:
+                
+                    return {
+                        "status": "success",
+                        "answer": "Not found in the provided lecture material.",
+                        "citations": {}
+                    }
+
+                # --- Generation ---
+                context_chunks = [
+                    f"[{i+1}] {doc['text']}"
+                    for i, doc in enumerate(retrieved_docs)
+                ]
+
+                try:  
+                    answer = await asyncio.wait_for(
+                        self.llm.generate(question, context_chunks),
+                        timeout=20
+                    )
+                except asyncio.TimeoutError:
+                    logger.warning("LLM timout")
+                    return {
+                        "status" : "error",
+                        "error_type": "LLMTimeout",
+                        "message" : "LLM generation timed out"
+                    }
+            
+                
+                # --- Citations ---
+                citations = {
+                    str(i+1): doc["citation"]
+                    for i, doc in enumerate(retrieved_docs)
+                }
+
+
+                # --- Logging ---
+                
+                return {
+                    "status": "success",
+                    "answer": answer,
+                    "citations": citations
+                }
+
+            except Exception as e:
+                
+                logger.exception("RAG failure")
+                return {
+                    "status": "error",
+                    "error_type": "RAGRuntimeError",
+                    "message": "Internal RAG error"
+                }

src/training/run_ingestion.py

@@ -0,0 +1,74 @@
+from pathlib import Path
+import json
+from typing import Union, List
+
+from src.ingestion.document_loader import load_documents
+from src.preprocessing.chunker_markdown import chunk_document
+from src.embedding.embeddings import EmbeddingModel
+from src.vector_store.FAISS_store import VectorStore
+from src.mlops.tracking import *
+import faiss
+
+def run_ingestion(
+    data_path: Path,
+    chunk_size: int = 400,
+    overlap: int = 50,
+    model_name: str = "paraphrase-multilingual-MiniLM-L12-v2"
+):
+
+
+    start_experiment("rag_ingestion")
+
+    documents = load_documents(data_path)
+
+    all_chunks = []
+    all_metadatas = []
+
+    chunk_id = 0
+    for doc in documents:
+        chunks = chunk_document(
+            doc["text"],
+            chunk_size=chunk_size,
+            overlap=overlap
+        )
+
+        for chunk in chunks:
+            all_chunks.append(chunk)
+            all_metadatas.append({
+                "source": doc["metadata"]["source"],
+                "page": doc["metadata"]["page"],
+                "text": chunk,
+                "chunk_id": chunk_id
+            })
+            chunk_id += 1
+
+    embedder = EmbeddingModel()
+    vectors = embedder.encode(all_chunks)
+
+    store = VectorStore(dim=vectors.shape[1])
+    store.add(vectors, all_metadatas)
+
+    
+    with open("metadata.json", "w", encoding="utf-8") as f:
+        json.dump(store.metadata, f, ensure_ascii=False, indent=2)
+
+    faiss.write_index(store.index, "faiss.index")
+
+    log_params_dict({   
+        "chunk_size": chunk_size,
+        "overlap": overlap,
+        "embedding_model": model_name
+    })
+
+    log_metrics_dict({
+        "num_chunks": len(all_chunks),
+        "embedding_dim": vectors.shape[1]
+    })
+
+    log_artifact_metadatas("metadata.json")
+    end_run()
+
+    return {
+        "num_docs": len(documents),
+        "num_chunks": len(all_chunks)
+    }

src/vector_store/FAISS_store.py

@@ -0,0 +1,34 @@
+import faiss 
+import numpy as np
+from typing import List , Dict
+
+class VectorStore:
+
+    def __init__(self , dim : int):
+        self.index = faiss.IndexFlatIP(dim)
+        self.metadata : List[Dict] = []
+
+    def add(self , vectors: np.ndarray , metadatas:List[Dict]):
+        self.index.add(vectors)
+        self.metadata.extend(metadatas)
+
+    def search(self, query_vector : np.ndarray , k :int = 5):
+        scores, indices = self.index.search(query_vector , k)
+
+        results = []
+
+        for idx , score in zip(indices[0], scores[0]):
+            if idx == -1 :
+                continue
+            meta = self.metadata[idx]
+
+            results.append({
+                "score": float(score),
+                "text": meta["text"],           
+                "citation": {
+                    "source": meta["source"],
+                    "page": meta["page"],
+                }
+            })
+        return results
+