Upload multimodal-rag-system model from Hamza66628/multimodal-rag-system

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+image_index.faiss filter=lfs diff=lfs merge=lfs -text
+text_index.faiss filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,44 @@
+---
+license: mit
+tags:
+- rag
+- multimodal
+- faiss
+- sentence-transformers
+- clip
+- mistral
+- information-retrieval
+---
+
+# Multimodal RAG System
+
+This repository contains a complete Multimodal Retrieval-Augmented Generation (RAG) system that combines text and image search with LLM-based answer generation.
+
+## System Components
+
+- **Text Embeddings**: Sentence-BERT (all-MiniLM-L6-v2) - 384 dimensions
+- **Image Embeddings**: CLIP (ViT-B/32) - 512 dimensions
+- **Vector Database**: FAISS indices for efficient similarity search
+- **LLM**: Mistral-7B-Instruct (4-bit quantized)
+- **Total Vectors**: 446 (161 text + 285 images)
+
+## Files
+
+- `text_index.faiss`: FAISS index for text embeddings
+- `image_index.faiss`: FAISS index for image embeddings
+- `text_metadata.pkl`: Metadata for text chunks (source, page, content)
+- `image_metadata.pkl`: Metadata for images (source, page, image_id)
+- `config.json`: System configuration
+- `image_summary.json`: Reference summary of images
+
+## Usage
+
+See the load cells in the notebook for loading and using this RAG system.
+
+## Features
+
+- Semantic text search
+- Cross-modal image search (text query → image results)
+- Multiple prompting strategies (Standard, Chain-of-Thought, Few-shot, Zero-shot)
+- Source attribution and traceability
+- Real-time answer generation

config.json

@@ -0,0 +1,10 @@
+{
+  "text_embedding_dim": 384,
+  "image_embedding_dim": 512,
+  "text_model": "sentence-transformers/all-MiniLM-L6-v2",
+  "image_model": "openai/clip-vit-base-patch32",
+  "llm_model": "mistralai/Mistral-7B-Instruct-v0.2",
+  "num_text_vectors": 161,
+  "num_image_vectors": 285,
+  "total_vectors": 446
+}

image_index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16b0bf902d1c151c553badfd7a83b4d2eacddf0dc10c7e8db9320b0d68097ddf
+size 583725

image_metadata.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e8b4d9d33bbc93a3044a6e13471608f858090062a561c001bd25e62fb3556ba
+size 33409

image_summary.json

@@ -0,0 +1,502 @@
+[
+  {
+    "image_id": "3. FYP-Handbook-2023_p2_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 2
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p2_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 2
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p2_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 2
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p2_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 2
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p4_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 4
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p4_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 4
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p4_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 4
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p4_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 4
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p5_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 5
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p5_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 5
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p5_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 5
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p5_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 5
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p6_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 6
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p6_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 6
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p6_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 6
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p6_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 6
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p7_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 7
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p7_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 7
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p7_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 7
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p7_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 7
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p8_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 8
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p8_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 8
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p8_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 8
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p8_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 8
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p9_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 9
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p9_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 9
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p9_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 9
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p9_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 9
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p10_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 10
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p10_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 10
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p10_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 10
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p10_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 10
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p11_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 11
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p11_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 11
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p11_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 11
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p11_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 11
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p12_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 12
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p12_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 12
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p12_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 12
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p12_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 12
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p13_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 13
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p13_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 13
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p13_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 13
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p13_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 13
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p14_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 14
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p14_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 14
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p14_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 14
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p14_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 14
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p15_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 15
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p15_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 15
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p15_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 15
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p15_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 15
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p16_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 16
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p16_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 16
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p16_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 16
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p16_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 16
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p17_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 17
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p17_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 17
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p17_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 17
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p17_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 17
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p18_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 18
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p18_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 18
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p18_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 18
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p18_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 18
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p19_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 19
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p19_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 19
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p19_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 19
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p19_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 19
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p20_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 20
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p20_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 20
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p20_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 20
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p20_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 20
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p21_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 21
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p21_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 21
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p21_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 21
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p21_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 21
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p22_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 22
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p22_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 22
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p22_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 22
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p22_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 22
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p23_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 23
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p23_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 23
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p23_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 23
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p23_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 23
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p24_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 24
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p24_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 24
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p24_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 24
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p24_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 24
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p25_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 25
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p25_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 25
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p25_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 25
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p25_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 25
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p26_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 26
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p26_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 26
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p26_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 26
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p26_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 26
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p27_img0",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 27
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p27_img1",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 27
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p27_img2",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 27
+  },
+  {
+    "image_id": "3. FYP-Handbook-2023_p27_img3",
+    "source": "3. FYP-Handbook-2023.pdf",
+    "page": 27
+  }
+]

text_index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f1fcb2f226e86cb2d4ed0a58201b2660cd04ccee45c98ca47060e6bce6a4244
+size 247341

text_metadata.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cda880e85a5b62610c296c9e515c0ea9a208ab625bc7e93324aa074993990d39
+size 250958