Fix HF persistent storage paths

.gitignore

@@ -0,0 +1,5 @@
+data/
+outputs/
+uploads/
+__pycache__/
+*.pyc

Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.10-slim
+
+WORKDIR /code
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    poppler-utils \
+    libgl1 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,10 +0,0 @@
----
-title: ThesisBackend
-emoji: 🌖
-colorFrom: green
-colorTo: red
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app/api/routes/grading.py

@@ -0,0 +1,10 @@
+from fastapi import APIRouter, UploadFile, File
+from app.schemas.grading import GradingResponse
+from app.services.grading_service import grade_annotation
+
+router = APIRouter(prefix="/grade-annotation", tags=["Grading"])
+
+
+@router.post("/", response_model=GradingResponse)
+def grade(file: UploadFile = File(...)):
+    return grade_annotation(file)

app/api/routes/rag.py

@@ -0,0 +1,10 @@
+from fastapi import APIRouter
+from app.schemas.rag import AskRequest, AskResponse
+from app.services.rag_service import ask_question
+
+router = APIRouter(prefix="/rag", tags=["RAG"])
+
+
+@router.post("/ask", response_model=AskResponse)
+def ask(req: AskRequest):
+    return ask_question(req.question)

app/api/routes/upload.py

@@ -0,0 +1,10 @@
+from fastapi import APIRouter, UploadFile, File
+from app.schemas.upload import UploadResponse
+from app.services.ingestion_service import upload_pdf
+
+router = APIRouter(prefix="/upload-pdf", tags=["Ingestion"])
+
+
+@router.post("/", response_model=UploadResponse)
+def upload(file: UploadFile = File(...)):
+    return upload_pdf(file)

app/api/routes/visualize.py

@@ -0,0 +1,10 @@
+from fastapi import APIRouter
+from app.schemas.visualize import VisualizeRequest, VisualizeResponse
+from app.services.visualization_service import visualize
+
+router = APIRouter(prefix="/visualize", tags=["Visualization"])
+
+
+@router.post("/", response_model=VisualizeResponse)
+def run_visualize(req: VisualizeRequest):
+    return visualize(req.question)

app/main.py

@@ -0,0 +1,32 @@
+from fastapi import FastAPI
+from fastapi.staticfiles import StaticFiles
+from app.api.routes import rag
+from app.api.routes import rag, visualize
+from app.api.routes import rag, visualize, grading
+
+from app.api.routes import rag, visualize, grading, upload
+
+
+
+
+
+
+app = FastAPI(title="Multimodal RAG API")
+
+app.include_router(rag.router)
+
+app.include_router(visualize.router)
+
+app.include_router(grading.router)
+
+app.include_router(upload.router)
+
+
+
+
+app.mount("/outputs", StaticFiles(directory="outputs"), name="outputs")
+
+
+@app.get("/")
+def root():
+    return {"status": "running"}

app/schemas/grading.py

@@ -0,0 +1,8 @@
+from pydantic import BaseModel
+from typing import List, Optional
+
+
+class GradingResponse(BaseModel):
+    score: Optional[float]
+    feedback: str
+    missing_structures: List[str]

app/schemas/rag.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel
+from typing import List, Optional
+
+
+class AskRequest(BaseModel):
+    question: str
+
+
+class AskResponse(BaseModel):
+    answer: str
+    images: Optional[List[str]] = None

app/schemas/upload.py

@@ -0,0 +1,9 @@
+from pydantic import BaseModel
+from typing import Dict
+
+
+class UploadResponse(BaseModel):
+    status: str
+    message: str
+    text_ingestion: Dict
+    image_extraction: Dict

app/schemas/visualize.py

@@ -0,0 +1,10 @@
+from pydantic import BaseModel
+from typing import Optional
+
+
+class VisualizeRequest(BaseModel):
+    question: str
+
+
+class VisualizeResponse(BaseModel):
+    annotated_image: Optional[str]

app/services/grading_service.py

@@ -0,0 +1,30 @@
+import os
+from src.assessment.run_annotation_check import run_annotation_grading
+
+UPLOAD_DIR = "uploads"
+os.makedirs(UPLOAD_DIR, exist_ok=True)
+
+
+def grade_annotation(file):
+
+    file_path = os.path.join(UPLOAD_DIR, file.filename)
+
+    with open(file_path, "wb") as f:
+        f.write(file.file.read())
+
+    raw_result = run_annotation_grading(file_path)
+
+    # 🔥 If your grader returns a STRING → convert to structured format
+    if isinstance(raw_result, str):
+        return {
+            "score": None,
+            "feedback": raw_result,
+            "missing_structures": []
+        }
+
+    # 🔥 If your grader already returns a dict → pass through safely
+    return {
+        "score": raw_result.get("score"),
+        "feedback": raw_result.get("feedback", ""),
+        "missing_structures": raw_result.get("missing_structures", [])
+    }

app/services/ingestion_service.py

@@ -0,0 +1,31 @@
+import os
+from pathlib import Path
+from src.config.settings import settings
+from src.ingestion.run import run_ingestion
+from src.ingestion.run_image_extraction import run_image_extraction
+from src.multimodal.run_multimodal_rag import reload_rag
+
+
+UPLOAD_DIR = Path(settings.raw_data_dir)
+UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def upload_pdf(file):
+
+    file_path = UPLOAD_DIR / file.filename
+
+    with open(file_path, "wb") as f:
+        f.write(file.file.read())
+
+    text_result = run_ingestion()
+    image_result = run_image_extraction()
+
+    reload_rag()
+
+
+    return {
+        "status": "success",
+        "message": f"{file.filename} ingested and RAG reloaded",
+        "text_ingestion": text_result,
+        "image_extraction": image_result,
+    }

app/services/rag_service.py

@@ -0,0 +1,11 @@
+from src.multimodal.run_multimodal_rag import run_multimodal_rag
+
+
+def ask_question(question: str):
+
+    result = run_multimodal_rag(question)
+
+    return {
+        "answer": result["answer"],
+        "images": result.get("images", [])
+    }

app/services/visualization_service.py

@@ -0,0 +1,10 @@
+from src.visualization.run_visual_answer import run_visual_answer
+
+
+def visualize(question: str):
+
+    result = run_visual_answer(question)
+
+    return {
+        "annotated_image": result["annotated_image"]
+    }

multimodal_rag_thesis.egg-info/PKG-INFO

@@ -0,0 +1,26 @@
+Metadata-Version: 2.4
+Name: multimodal-rag-thesis
+Version: 0.1.0
+Summary: Add your description here
+Requires-Python: <3.12,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: chromadb>=1.5.0
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: langchain>=0.2.0
+Requires-Dist: langchain-chroma>=0.1.0
+Requires-Dist: langchain-community>=0.2.0
+Requires-Dist: langchain-huggingface>=0.0.3
+Requires-Dist: pymupdf
+Requires-Dist: pillow
+Requires-Dist: matplotlib
+Requires-Dist: streamlit
+Requires-Dist: onnxruntime<1.17
+Requires-Dist: opencv-python-headless<4.9
+Requires-Dist: sentence-transformers==2.7.0
+Requires-Dist: numpy<2
+Requires-Dist: torch==2.2.2
+Requires-Dist: langchain-groq>=1.1.2
+Requires-Dist: fastapi>=0.129.0
+Requires-Dist: uvicorn>=0.40.0
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: python-multipart>=0.0.22

multimodal_rag_thesis.egg-info/SOURCES.txt

@@ -0,0 +1,52 @@
+README.md
+pyproject.toml
+app/main.py
+app/api/routes/grading.py
+app/api/routes/rag.py
+app/api/routes/visualize.py
+app/schemas/grading.py
+app/schemas/rag.py
+app/schemas/visualize.py
+app/services/grading_service.py
+app/services/rag_service.py
+app/services/visualization_service.py
+multimodal_rag_thesis.egg-info/PKG-INFO
+multimodal_rag_thesis.egg-info/SOURCES.txt
+multimodal_rag_thesis.egg-info/dependency_links.txt
+multimodal_rag_thesis.egg-info/requires.txt
+multimodal_rag_thesis.egg-info/top_level.txt
+src/__init__.py
+src/main.py
+src/assessment/annotation_grader.py
+src/assessment/image_query_retriever.py
+src/assessment/label_extractor.py
+src/assessment/run_annotation_check.py
+src/config/__init__.py
+src/config/settings.py
+src/embeddings/__init__.py
+src/embeddings/embedding_factory.py
+src/ingestion/__init__.py
+src/ingestion/image_extractor.py
+src/ingestion/loader.py
+src/ingestion/run.py
+src/ingestion/run_image_extraction.py
+src/llm/__init__.py
+src/llm/llm_factory.py
+src/multimodal/__init__.py
+src/multimodal/clip_embedding.py
+src/multimodal/multimodal_indexer.py
+src/multimodal/multimodal_rag_chain.py
+src/multimodal/multimodal_retriever.py
+src/multimodal/run_multimodal_indexing.py
+src/multimodal/run_multimodal_query.py
+src/multimodal/run_multimodal_rag.py
+src/retrieval/__init__.py
+src/retrieval/query.py
+src/retrieval/rag_query.py
+src/retrieval/vector_store.py
+src/utils/__init__.py
+src/utils/logger.py
+src/visualization/annotation_schema.py
+src/visualization/image_annotator.py
+src/visualization/llm_structure_extractor.py
+src/visualization/run_visual_answer.py

multimodal_rag_thesis.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

multimodal_rag_thesis.egg-info/requires.txt

@@ -0,0 +1,20 @@
+chromadb>=1.5.0
+python-dotenv>=1.0.1
+langchain>=0.2.0
+langchain-chroma>=0.1.0
+langchain-community>=0.2.0
+langchain-huggingface>=0.0.3
+pymupdf
+pillow
+matplotlib
+streamlit
+onnxruntime<1.17
+opencv-python-headless<4.9
+sentence-transformers==2.7.0
+numpy<2
+torch==2.2.2
+langchain-groq>=1.1.2
+fastapi>=0.129.0
+uvicorn>=0.40.0
+pydantic>=2.12.5
+python-multipart>=0.0.22

multimodal_rag_thesis.egg-info/top_level.txt

@@ -0,0 +1,5 @@
+app
+data
+outputs
+src
+uploads

pyproject.toml

@@ -0,0 +1,38 @@
+[project]
+name = "multimodal-rag-thesis"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10,<3.12"
+
+
+dependencies = [
+    "chromadb>=1.5.0",
+    "python-dotenv>=1.0.1",
+    "langchain>=0.2.0",
+    "langchain-chroma>=0.1.0",
+    "langchain-community>=0.2.0",
+    "langchain-huggingface>=0.0.3",
+    "pymupdf",
+    "pillow",
+    "matplotlib",
+    "streamlit",
+    "onnxruntime<1.17",
+    "opencv-python-headless<4.9",
+    "sentence-transformers==2.7.0",
+    "numpy<2",
+    "torch==2.2.2",
+    "langchain-groq>=1.1.2",
+    "fastapi>=0.129.0",
+    "uvicorn>=0.40.0",
+    "pydantic>=2.12.5",
+    "python-multipart>=0.0.22",
+]
+
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["."]

requirements.txt

@@ -0,0 +1,160 @@
+aiohappyeyeballs==2.6.1
+aiohttp==3.13.3
+aiosignal==1.4.0
+altair==6.0.0
+annotated-doc==0.0.4
+annotated-types==0.7.0
+anyio==4.12.1
+async-timeout==4.0.3
+attrs==25.4.0
+backoff==2.2.1
+bcrypt==5.0.0
+blinker==1.9.0
+build==1.4.0
+cachetools==6.2.6
+certifi==2026.1.4
+charset-normalizer==3.4.4
+chromadb==1.5.0
+click==8.3.1
+coloredlogs==15.0.1
+contourpy==1.3.2
+cycler==0.12.1
+dataclasses-json==0.6.7
+distro==1.9.0
+durationpy==0.10
+exceptiongroup==1.3.1
+fastapi==0.129.0
+filelock==3.21.2
+flatbuffers==25.12.19
+fonttools==4.61.1
+frozenlist==1.8.0
+fsspec==2026.2.0
+gitdb==4.0.12
+gitpython==3.1.46
+googleapis-common-protos==1.72.0
+greenlet==3.3.1
+groq==0.37.1
+grpcio==1.78.0
+h11==0.16.0
+hf-xet==1.2.0
+httpcore==1.0.9
+httptools==0.7.1
+httpx==0.28.1
+httpx-sse==0.4.3
+huggingface-hub==0.36.2
+humanfriendly==10.0
+idna==3.11
+importlib-metadata==8.7.1
+importlib-resources==6.5.2
+jinja2==3.1.6
+joblib==1.5.3
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.26.0
+jsonschema-specifications==2025.9.1
+kiwisolver==1.4.9
+kubernetes==35.0.0
+langchain==1.2.10
+langchain-chroma==1.1.0
+langchain-classic==1.0.1
+langchain-community==0.4.1
+langchain-core==1.2.12
+langchain-groq==1.1.2
+langchain-huggingface==1.2.0
+langchain-text-splitters==1.1.0
+langgraph==1.0.8
+langgraph-checkpoint==4.0.0
+langgraph-prebuilt==1.0.7
+langgraph-sdk==0.3.5
+langsmith==0.7.1
+markdown-it-py==4.0.0
+markupsafe==3.0.3
+marshmallow==3.26.2
+matplotlib==3.10.8
+mdurl==0.1.2
+mmh3==5.2.0
+mpmath==1.3.0
+multidict==6.7.1
+-e file:///Users/human1/Documents/Thesis/POC2/phase1/multimodal-rag-thesis
+mypy-extensions==1.1.0
+narwhals==2.16.0
+networkx==3.4.2
+numpy==1.26.4
+oauthlib==3.3.1
+onnxruntime==1.16.3
+opencv-python-headless==4.8.1.78
+opentelemetry-api==1.39.1
+opentelemetry-exporter-otlp-proto-common==1.39.1
+opentelemetry-exporter-otlp-proto-grpc==1.39.1
+opentelemetry-proto==1.39.1
+opentelemetry-sdk==1.39.1
+opentelemetry-semantic-conventions==0.60b1
+orjson==3.11.7
+ormsgpack==1.12.2
+overrides==7.7.0
+packaging==26.0
+pandas==2.3.3
+pillow==12.1.1
+posthog==5.4.0
+propcache==0.4.1
+protobuf==6.33.5
+pyarrow==23.0.0
+pybase64==1.4.3
+pydantic==2.12.5
+pydantic-core==2.41.5
+pydantic-settings==2.12.0
+pydeck==0.9.1
+pygments==2.19.2
+pymupdf==1.27.1
+pyparsing==3.3.2
+pypika==0.51.1
+pyproject-hooks==1.2.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.2.1
+python-multipart==0.0.22
+pytz==2025.2
+pyyaml==6.0.3
+referencing==0.37.0
+regex==2026.1.15
+requests==2.32.5
+requests-oauthlib==2.0.0
+requests-toolbelt==1.0.0
+rich==14.3.2
+rpds-py==0.30.0
+safetensors==0.7.0
+scikit-learn==1.7.2
+scipy==1.15.3
+sentence-transformers==2.7.0
+shellingham==1.5.4
+six==1.17.0
+smmap==5.0.2
+sniffio==1.3.1
+sqlalchemy==2.0.46
+starlette==0.52.1
+streamlit==1.54.0
+sympy==1.14.0
+tenacity==9.1.4
+threadpoolctl==3.6.0
+tokenizers==0.22.2
+toml==0.10.2
+tomli==2.4.0
+torch==2.2.2
+tornado==6.5.4
+tqdm==4.67.3
+transformers==4.57.6
+typer==0.23.1
+typing-extensions==4.15.0
+typing-inspect==0.9.0
+typing-inspection==0.4.2
+tzdata==2025.3
+urllib3==2.6.3
+uuid-utils==0.14.0
+uvicorn==0.40.0
+uvloop==0.22.1
+watchfiles==1.1.1
+websocket-client==1.9.0
+websockets==16.0
+xxhash==3.6.0
+yarl==1.22.0
+zipp==3.23.0
+zstandard==0.25.0

src/assessment/annotation_grader.py

@@ -0,0 +1,29 @@
+from src.llm.llm_factory import get_llm
+
+
+class AnnotationGrader:
+
+    def __init__(self):
+        self.llm = get_llm()
+
+    def grade(self, user_labels, reference_text):
+
+        prompt = f"""
+Compare the student labels with the reference anatomy.
+
+Student labels:
+{user_labels}
+
+Reference:
+{reference_text}
+
+Return:
+
+Correct:
+Missing:
+Incorrect:
+"""
+
+        response = self.llm.invoke(prompt)
+
+        return response.content

src/assessment/image_query_retriever.py

@@ -0,0 +1,26 @@
+from src.multimodal.clip_embedding import CLIPEmbedding
+from langchain_chroma import Chroma
+from src.config.settings import settings
+
+
+class ImageQueryRetriever:
+
+    def __init__(self):
+
+        self.embedding = CLIPEmbedding()
+
+        self.vectorstore = Chroma(
+            collection_name="multimodal_rag",
+            persist_directory=f"{settings.processed_data_dir}/multimodal_chroma"
+        )
+
+    def retrieve_similar(self, image_path, k=1):
+
+        emb = self.embedding.embed_image([image_path])[0]
+
+        results = self.vectorstore._collection.query(
+            query_embeddings=[emb.tolist()],
+            n_results=k
+        )
+
+        return results["metadatas"][0]

src/assessment/label_extractor.py

@@ -0,0 +1,23 @@
+from src.llm.llm_factory import get_llm
+
+
+class LabelExtractor:
+
+    def __init__(self):
+        self.llm = get_llm()
+
+    def extract(self, image_path):
+
+        prompt = f"""
+The user uploaded an annotated anatomy image.
+
+List the anatomical labels present in the image.
+
+Return JSON:
+
+{{ "labels": ["label1", "label2"] }}
+"""
+
+        response = self.llm.invoke(prompt)
+
+        return response.content

src/assessment/run_annotation_check.py

@@ -0,0 +1,35 @@
+from src.assessment.image_query_retriever import ImageQueryRetriever
+from src.assessment.label_extractor import LabelExtractor
+from src.assessment.annotation_grader import AnnotationGrader
+
+
+# 🔥 Global instances for API performance
+retriever = ImageQueryRetriever()
+extractor = LabelExtractor()
+grader = AnnotationGrader()
+
+
+def run_annotation_grading(image_path: str):
+    """
+    FastAPI entry point for grading a student annotation.
+    """
+
+    reference = retriever.retrieve_similar(image_path)
+
+    user_labels = extractor.extract(image_path)
+
+    result = grader.grade(user_labels, reference)
+
+    return result
+
+
+def main():
+    image_path = input("Enter path to annotated image: ")
+
+    result = run_annotation_grading(image_path)
+
+    print("\nRESULT:\n", result)
+
+
+if __name__ == "__main__":
+    main()

src/config/settings.py

@@ -0,0 +1,31 @@
+from dataclasses import dataclass
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+
+# 🔥 This becomes /data on Hugging Face, and stays local when developing
+BASE_DATA_DIR = os.getenv("HF_HOME", "data")
+
+
+@dataclass
+class Settings:
+    app_env: str = os.getenv("APP_ENV", "development")
+    log_level: str = os.getenv("LOG_LEVEL", "INFO")
+
+    # 📂 Data paths
+    base_data_dir: str = BASE_DATA_DIR
+    raw_data_dir: str = os.path.join(BASE_DATA_DIR, "raw")
+    processed_data_dir: str = os.path.join(BASE_DATA_DIR, "processed")
+    chroma_dir: str = os.path.join(BASE_DATA_DIR, "chroma")
+
+    # 🤖 Models
+    embedding_model: str = "sentence-transformers/all-MiniLM-L6-v2"
+    llm_model: str = "llama-3.1-8b-instant"
+
+
+settings = Settings()
+
+
+
+

src/embeddings/embedding_factory.py

@@ -0,0 +1,8 @@
+from langchain_huggingface import HuggingFaceEmbeddings
+from src.config.settings import settings
+
+
+def get_text_embedding():
+    return HuggingFaceEmbeddings(
+        model_name=settings.embedding_model
+    )

src/ingestion/image_extractor.py

@@ -0,0 +1,100 @@
+import fitz  # PyMuPDF
+import json
+from pathlib import Path
+
+from src.config.settings import settings
+from src.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ImageExtractor:
+
+    def __init__(self):
+        self.output_dir = Path(settings.processed_data_dir) / "images"
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        self.metadata = []
+
+        # 🚫 pages we never want (publisher / front matter / credits)
+        self.page_noise_keywords = [
+            "learning resources",
+            "about our team",
+            "senior contributors",
+            "powerpoint slides",
+            "pronunciation guide",
+            "acknowledgments",
+            "reviewers",
+            "openstax",
+        ]
+
+    def extract_from_pdf(self, pdf_path: Path):
+
+        logger.info(f"Extracting images from {pdf_path.name}")
+
+        doc = fitz.open(pdf_path)
+
+        for page_index in range(len(doc)):
+
+            page = doc[page_index]
+            page_text = page.get_text("text")
+            text_lower = page_text.lower()
+
+            # 🚫 Skip non-content pages
+            if any(keyword in text_lower for keyword in self.page_noise_keywords):
+                continue
+
+            image_list = page.get_images(full=True)
+
+            if not image_list:
+                continue
+
+            for img_index, img in enumerate(image_list):
+
+                xref = img[0]
+                base_image = doc.extract_image(xref)
+                image_bytes = base_image["image"]
+
+                pix = fitz.Pixmap(doc, xref)
+
+                # ✅ Skip tiny images (logos, bullets, icons)
+                if pix.width < 200 or pix.height < 200:
+                    continue
+
+                # ✅ Skip very low file size
+                if len(image_bytes) < 20_000:
+                    continue
+
+                image_name = (
+                    f"{pdf_path.stem}_page_{page_index+1}_img_{img_index}.png"
+                )
+
+                image_path = self.output_dir / image_name
+
+                with open(image_path, "wb") as f:
+                    f.write(image_bytes)
+
+                self.metadata.append(
+                    {
+                        "image_path": str(image_path),
+                        "page": page_index + 1,
+                        "source": pdf_path.name,
+                        "image_index": img_index,
+                        "width": pix.width,
+                        "height": pix.height,
+                        "nearby_text": page_text[:1000],
+                    }
+                )
+
+                pix = None  # free memory
+
+        doc.close()
+
+    def save_metadata(self):
+
+        metadata_path = Path(settings.processed_data_dir) / "image_metadata.json"
+
+        with open(metadata_path, "w") as f:
+            json.dump(self.metadata, f, indent=2)
+
+        logger.info(f"Saved image metadata → {metadata_path}")

src/ingestion/loader.py

@@ -0,0 +1,84 @@
+from pathlib import Path
+from langchain_community.document_loaders import PyMuPDFLoader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+from src.config.settings import settings
+from src.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class DocumentLoader:
+
+    def load_pdfs(self):
+        data_path = Path(settings.raw_data_dir)
+        pdf_files = list(data_path.glob("*.pdf"))
+
+        if not pdf_files:
+            logger.warning("No PDFs found in data/raw")
+            return []
+
+        documents = []
+
+        for pdf in pdf_files:
+            logger.info(f"Loading PDF: {pdf.name}")
+            loader = PyMuPDFLoader(str(pdf))
+
+            pages = loader.load()
+
+            clean_pages = []
+
+            for page in pages:
+                text = page.page_content.strip().lower()
+
+                # 🚫 remove index pages
+                if "index" in text[:200]:
+                    continue
+
+                # 🚫 remove table of contents
+                if "chapter" in text and "...." in text:
+                    continue
+
+                # 🚫 remove glossary-style alphabetical lists
+                if text.count(",") > 20 and len(text) < 1500:
+                    continue
+
+                clean_pages.append(page)
+
+            logger.info(f"Kept {len(clean_pages)} useful pages.")
+
+            documents.extend(clean_pages)
+
+        logger.info(f"Total kept pages: {len(documents)}")
+
+        return documents
+
+
+
+    def split_documents(self, documents):
+
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=800,
+            chunk_overlap=150
+        )
+
+        chunks = text_splitter.split_documents(documents)
+
+        filtered_chunks = []
+
+        for chunk in chunks:
+            text = chunk.page_content.strip()
+
+            # Remove very short chunks
+            if len(text) < 200:
+                continue
+
+            # Remove index/table-of-contents style chunks
+            if text.count(".....") > 2:
+                continue
+
+            filtered_chunks.append(chunk)
+
+        logger.info(f"Split into {len(filtered_chunks)} clean chunks.")
+
+        return filtered_chunks

src/ingestion/run.py

@@ -0,0 +1,38 @@
+from src.ingestion.loader import DocumentLoader
+from src.embeddings.embedding_factory import get_text_embedding
+from src.retrieval.vector_store import VectorStoreFactory
+from src.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def run_ingestion():
+    logger.info("Starting ingestion pipeline...")
+
+    loader = DocumentLoader()
+
+    documents = loader.load_pdfs()
+
+    if not documents:
+        logger.warning("No documents to ingest.")
+        return {"status": "warning", "message": "No documents found"}
+
+    chunks = loader.split_documents(documents)
+
+    embedding = get_text_embedding()
+
+    vectordb = VectorStoreFactory.create(embedding)
+
+    vectordb.add_documents(chunks)
+
+    logger.info("Ingestion complete.")
+
+    return {"status": "success", "message": "Text ingestion complete"}
+
+
+def main():
+    run_ingestion()
+
+
+if __name__ == "__main__":
+    main()

src/ingestion/run_image_extraction.py

@@ -0,0 +1,35 @@
+from pathlib import Path
+from src.config.settings import settings
+from src.ingestion.image_extractor import ImageExtractor
+from src.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def run_image_extraction():
+
+    raw_path = Path(settings.raw_data_dir)
+    pdf_files = list(raw_path.glob("*.pdf"))
+
+    if not pdf_files:
+        logger.warning("No PDFs found.")
+        return {"status": "warning", "message": "No PDFs for image extraction"}
+
+    extractor = ImageExtractor()
+
+    for pdf in pdf_files:
+        extractor.extract_from_pdf(pdf)
+
+    extractor.save_metadata()
+
+    logger.info("Image extraction complete.")
+
+    return {"status": "success", "message": "Image extraction complete"}
+
+
+def main():
+    run_image_extraction()
+
+
+if __name__ == "__main__":
+    main()

src/llm/llm_factory.py

@@ -0,0 +1,17 @@
+import os
+from langchain_groq import ChatGroq
+from src.config.settings import settings
+from src.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def get_llm():
+
+    logger.info(f"Loading Groq model: {settings.llm_model}")
+
+    return ChatGroq(
+        model=settings.llm_model,
+        api_key=os.getenv("GROQ_API_KEY"),
+        temperature=0
+    )

src/main.py

@@ -0,0 +1,37 @@
+from src.config.settings import settings
+from src.utils.logger import get_logger
+from src.ingestion.loader import DocumentLoader
+from src.embeddings.embedding_factory import get_text_embedding
+from src.retrieval.vector_store import VectorStoreFactory
+from src.llm.llm_factory import get_llm
+
+
+import os
+
+os.makedirs(settings.raw_data_dir, exist_ok=True)
+os.makedirs(settings.processed_data_dir, exist_ok=True)
+os.makedirs(settings.chroma_dir, exist_ok=True)
+
+
+logger = get_logger(__name__)
+
+
+def main():
+    logger.info("Multimodal RAG system initialized.")
+    logger.info(f"Running in environment: {settings.app_env}")
+
+    loader = DocumentLoader()
+    loader.load()
+
+    embedding = get_text_embedding()
+    logger.info("Embedding model loaded.")
+
+    vectordb = VectorStoreFactory.create(embedding)
+
+    llm = get_llm()
+
+    logger.info("System setup complete.")
+
+
+if __name__ == "__main__":
+    main()

src/multimodal/__init__.py

@@ -0,0 +1,7 @@
+from .clip_embedding import CLIPEmbedding
+from .multimodal_indexer import MultimodalIndexer
+
+__all__ = [
+    "CLIPEmbedding",
+    "MultimodalIndexer",
+]

src/multimodal/clip_embedding.py

@@ -0,0 +1,17 @@
+from sentence_transformers import SentenceTransformer
+from PIL import Image
+
+
+class CLIPEmbedding:
+
+    def __init__(self):
+        self.model = SentenceTransformer("clip-ViT-B-32")
+
+    def embed_text(self, texts):
+        return self.model.encode(texts, convert_to_numpy=True)
+
+    def embed_image(self, image_paths):
+
+        images = [Image.open(p).convert("RGB") for p in image_paths]
+
+        return self.model.encode(images, convert_to_numpy=True)

src/multimodal/multimodal_indexer.py

@@ -0,0 +1,69 @@
+import json
+from pathlib import Path
+from langchain_chroma import Chroma
+
+from src.config.settings import settings
+from src.multimodal.clip_embedding import CLIPEmbedding
+from src.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class MultimodalIndexer:
+
+    def __init__(self):
+
+        self.embedding = CLIPEmbedding()
+
+        self.vectorstore = Chroma(
+            collection_name="multimodal_rag",
+            persist_directory=f"{settings.processed_data_dir}/multimodal_chroma"
+        )
+
+    def index_images(self):
+
+        metadata_path = Path(settings.processed_data_dir) / "image_metadata.json"
+
+        if not metadata_path.exists():
+            logger.warning("image_metadata.json not found.")
+            return
+
+        with open(metadata_path) as f:
+            metadata = json.load(f)
+
+        if not metadata:
+            logger.warning("No image metadata found.")
+            return
+
+        image_paths = [item["image_path"] for item in metadata]
+
+        logger.info(f"Embedding {len(image_paths)} images with CLIP...")
+
+        image_embeddings = self.embedding.embed_image(image_paths)
+
+        ids = []
+        documents = []
+        metadatas = []
+
+        for i, (emb, item) in enumerate(zip(image_embeddings, metadata)):
+
+            ids.append(f"image_{i}")
+
+            documents.append(item["nearby_text"])
+
+            metadatas.append({
+                "type": "image",
+                "image_path": item["image_path"],
+                "page": item["page"],
+                "source": item["source"],
+            })
+
+        self.vectorstore._collection.add(
+            embeddings=image_embeddings.tolist(),
+            documents=documents,
+            metadatas=metadatas,
+            ids=ids
+        )
+
+
+        logger.info("Image embeddings stored in Chroma.")

src/multimodal/multimodal_rag_chain.py

@@ -0,0 +1,36 @@
+from src.multimodal.multimodal_retriever import MultimodalRetriever
+from src.llm.llm_factory import get_llm
+
+
+class MultimodalRAG:
+
+    def __init__(self):
+
+        self.retriever = MultimodalRetriever()
+        self.llm = get_llm()
+
+    def ask(self, query):
+
+        docs, metas = self.retriever.retrieve(query, k=5)
+
+        context = "\n\n".join(docs)
+
+        prompt = f"""
+You are a medical anatomy assistant.
+
+Use the context to answer the question.
+
+Context:
+{context}
+
+Question:
+{query}
+"""
+
+        response = self.llm.invoke(prompt)
+
+        image_paths = [
+            m["image_path"] for m in metas if m["type"] == "image"
+        ]
+
+        return response.content, image_paths

src/multimodal/multimodal_retriever.py

@@ -0,0 +1,29 @@
+from langchain_chroma import Chroma
+from src.config.settings import settings
+from src.multimodal.clip_embedding import CLIPEmbedding
+
+
+class MultimodalRetriever:
+
+    def __init__(self):
+
+        self.embedding = CLIPEmbedding()
+
+        self.vectorstore = Chroma(
+            collection_name="multimodal_rag",
+            persist_directory=f"{settings.processed_data_dir}/multimodal_chroma"
+        )
+
+    def retrieve(self, query, k=5):
+
+        query_embedding = self.embedding.embed_text([query])[0]
+
+        results = self.vectorstore._collection.query(
+            query_embeddings=[query_embedding.tolist()],
+            n_results=k
+        )
+
+        documents = results["documents"][0]
+        metadatas = results["metadatas"][0]
+
+        return documents, metadatas

src/multimodal/run_multimodal_indexing.py

@@ -0,0 +1,11 @@
+from src.multimodal.multimodal_indexer import MultimodalIndexer
+
+
+def main():
+
+    indexer = MultimodalIndexer()
+    indexer.index_images()
+
+
+if __name__ == "__main__":
+    main()