Uploading Mulitimodal Retrieval Augmented Generation System.

app.py

@@ -6,8 +6,8 @@ import hashlib
 import torch
 import gradio as gr
 
-from setup.multimodal_rag.model import embedding_model, model, processor
-from setup.multimodal_rag.main import preprocess_pdf, semantic_search, generate_answer_stream
+from model_setup import embedding_model, model, processor
+from main import preprocess_pdf, semantic_search, generate_answer_stream
 
 device = "cuda" if torch.cuda.is_available() else "cpu"
 

main.py

@@ -16,13 +16,13 @@ from PIL import Image
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from transformers import TextIteratorStreamer
 
-from setup.multimodal_rag.utils import (
+from utils import (
     save_cache, load_cache,
     init_faiss_indexflatip, add_embeddings_to_index,
     search_faiss_index, save_faiss_index, load_faiss_index, cleanup_images
 )
 
-from setup.multimodal_rag.model import embedding_model, model, processor
+from model_setup import embedding_model, model, processor
 
 device = "cuda" if torch.cuda.is_available() else "cpu"
 

model_setup.py

@@ -0,0 +1,35 @@
+
+"""loading the models to be used by the Mulltimodal RAG system."""
+
+import torch
+
+from sentence_transformers import SentenceTransformer
+from transformers import AutoProcessor, Gemma3ForConditionalGeneration, BitsAndBytesConfig
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+# Embedding model
+embedding_model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+
+# Gemma3 quantized config
+model_name = "google/gemma-3-4b-it"
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=torch.bfloat16,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+)
+
+# Load Gemma3
+model = Gemma3ForConditionalGeneration.from_pretrained(
+    model_name,
+    torch_dtype=torch.bfloat16,
+    device_map="auto",
+    quantization_config=bnb_config,
+    low_cpu_mem_usage=True,
+    attn_implementation="sdpa"
+)
+model.eval()
+
+# Processor
+processor = AutoProcessor.from_pretrained(model_name, use_fast=True)