Build FAISS at runtime from HF dataset

.gitignore

@@ -2,3 +2,5 @@
 data/
 vectorstore/
 venv/
+*.faiss
+*.pkl

app.py

@@ -8,25 +8,45 @@ from langchain.chains import RetrievalQA
 from langchain_core.prompts import PromptTemplate
 from langchain_groq import ChatGroq
 
+from huggingface_hub import hf_hub_download
+from langchain.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
 load_dotenv()
 
-DB_FAISS_PATH = "vectorstore/db_faiss"
+HF_DATASET_REPO = "deepak-cse-jha/medibot-data"
+PDF_FILENAME = "The_GALE_ENCYCLOPEDIA_of_MEDICINE_SECOND.pdf"
+
 
 @st.cache_resource
 def get_vectorstore():
-    if not os.path.exists(DB_FAISS_PATH):
-        st.error("FAISS vectorstore not found")
-        st.stop()
+    # 1. Download PDF from HF Dataset
+    pdf_path = hf_hub_download(
+        repo_id=HF_DATASET_REPO,
+        filename=PDF_FILENAME,
+        repo_type="dataset"
+    )
+
+    # 2. Load PDF
+    loader = PyPDFLoader(pdf_path)
+    documents = loader.load()
+
+    # 3. Split text
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1000,
+        chunk_overlap=150
+    )
+    docs = splitter.split_documents(documents)
 
+    # 4. Create embeddings
     embeddings = HuggingFaceEmbeddings(
         model_name="sentence-transformers/all-MiniLM-L6-v2"
     )
 
-    return FAISS.load_local(
-        DB_FAISS_PATH,
-        embeddings,
-        allow_dangerous_deserialization=True
-    )
+    # 5. Build FAISS in memory
+    vectorstore = FAISS.from_documents(docs, embeddings)
+
+    return vectorstore
 
 
 def get_prompt():
@@ -63,7 +83,7 @@ def main():
     for msg in st.session_state.messages:
         st.chat_message(msg["role"]).markdown(msg["content"])
 
-    user_input = st.chat_input("Ask your question")
+    user_input = st.chat_input("Ask your medical question")
 
     if user_input:
         st.chat_message("user").markdown(user_input)
@@ -95,12 +115,8 @@ def main():
             {"role": "assistant", "content": answer}
         )
 
-        st.chat_message("assistant").markdown(
-            "Source Docs:\n\n" + str(sources)
-        )
-        st.session_state.messages.append(
-            {"role": "assistant", "content": str(sources)}
-        )
+        with st.expander("Source Documents"):
+            st.write(sources)
 
 
 if __name__ == "__main__":

requirements.txt

@@ -95,3 +95,4 @@ tzdata==2025.2; python_version >= '2'
 urllib3==2.5.0; python_version >= '3.9'
 yarl==1.20.1; python_version >= '3.9'
 zstandard==0.23.0; python_version >= '3.8'
+huggingface_hub

utils/connect_memory_with_llm.py

@@ -1,56 +1,62 @@
 import os
-
-from langchain_huggingface import HuggingFaceEndpoint
+from langchain_huggingface import HuggingFaceEndpoint, HuggingFaceEmbeddings
 from langchain_core.prompts import PromptTemplate
 from langchain.chains import RetrievalQA
-from langchain_huggingface import HuggingFaceEmbeddings
-from langchain_community.vectorstores import FAISS
 
-from dotenv import load_dotenv, find_dotenv
-load_dotenv(find_dotenv())
+from utils.create_faiss_from_dataset import get_or_create_faiss
+
 
+HF_TOKEN = os.environ.get("HF_TOKEN")
+HUGGINGFACE_REPO_ID = "mistralai/Mistral-7B-Instruct-v0.3"
 
-HF_TOKEN=os.environ.get("HF_TOKEN")
-HUGGINGFACE_REPO_ID="mistralai/Mistral-7B-Instruct-v0.3"
 
-def load_llm(huggingface_repo_id):
-    llm=HuggingFaceEndpoint(
-        repo_id=huggingface_repo_id,
+def load_llm(repo_id):
+    return HuggingFaceEndpoint(
+        repo_id=repo_id,
         temperature=0.5,
-        model_kwargs={"token":HF_TOKEN,
-                      "max_length":"512"}
+        model_kwargs={
+            "token": HF_TOKEN,
+            "max_length": 512
+        }
     )
-    return llm
 
 
 CUSTOM_PROMPT_TEMPLATE = """
 Use the pieces of information provided in the context to answer user's question.
-If you dont know the answer, just say that you dont know, dont try to make up an answer. 
-Dont provide anything out of the given context
+If you don't know the answer, say you don't know.
+Do not make up answers.
+
+Context:
+{context}
 
-Context: {context}
-Question: {question}
+Question:
+{question}
 
-Start the answer directly. No small talk please.
+Answer directly.
 """
 
-def set_custom_prompt(custom_prompt_template):
-    prompt=PromptTemplate(template=custom_prompt_template, input_variables=["context", "question"])
-    return prompt
-
-DB_FAISS_PATH= "../vectorstore/db_faiss"
-embedding_model=HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
-db=FAISS.load_local(DB_FAISS_PATH, embedding_model, allow_dangerous_deserialization=True)
-
-qa_chain=RetrievalQA.from_chain_type(
-    llm=load_llm(HUGGINGFACE_REPO_ID),
-    chain_type="stuff",
-    retriever=db.as_retriever(search_kwargs={'k':3}),
-    return_source_documents=True,
-    chain_type_kwargs={'prompt':set_custom_prompt(CUSTOM_PROMPT_TEMPLATE)}
-)
-
-user_query=input("Write Query Here: ")
-response=qa_chain.invoke({'query': user_query})
-print("RESULT: ", response["result"])
-print("SOURCE DOCUMENTS: ", response["source_documents"])
+
+def set_custom_prompt():
+    return PromptTemplate(
+        template=CUSTOM_PROMPT_TEMPLATE,
+        input_variables=["context", "question"]
+    )
+
+
+def get_qa_chain():
+    # ✅ Build or load FAISS at runtime
+    vectorstore = get_or_create_faiss()
+
+    retriever = vectorstore.as_retriever(search_kwargs={"k": 3})
+
+    qa_chain = RetrievalQA.from_chain_type(
+        llm=load_llm(HUGGINGFACE_REPO_ID),
+        chain_type="stuff",
+        retriever=retriever,
+        return_source_documents=True,
+        chain_type_kwargs={
+            "prompt": set_custom_prompt()
+        }
+    )
+
+    return qa_chain

utils/create_faiss_from_dataset.py

@@ -0,0 +1,58 @@
+import os
+from huggingface_hub import hf_hub_download
+from langchain_community.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+
+DATASET_REPO = "deepak-cse-jha/medibot-data"
+PDF_NAME = "The_GALE_ENCYCLOPEDIA_of_MEDICINE_SECOND.pdf"
+
+FAISS_DIR = "/tmp/faiss_index"
+PDF_PATH = "/tmp/medical.pdf"
+
+
+def get_or_create_faiss():
+    # 1️⃣ If FAISS already exists, load it
+    if os.path.exists(FAISS_DIR):
+        embeddings = HuggingFaceEmbeddings(
+            model_name="sentence-transformers/all-MiniLM-L6-v2"
+        )
+        return FAISS.load_local(
+            FAISS_DIR,
+            embeddings,
+            allow_dangerous_deserialization=True,
+        )
+
+    # 2️⃣ Download PDF from HF Dataset
+    hf_hub_download(
+        repo_id=DATASET_REPO,
+        filename=PDF_NAME,
+        repo_type="dataset",
+        local_dir="/tmp",
+        local_dir_use_symlinks=False,
+    )
+
+    os.rename(f"/tmp/{PDF_NAME}", PDF_PATH)
+
+    # 3️⃣ Load and split PDF
+    loader = PyPDFLoader(PDF_PATH)
+    documents = loader.load()
+
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1000,
+        chunk_overlap=200,
+    )
+    texts = splitter.split_documents(documents)
+
+    # 4️⃣ Create embeddings + FAISS
+    embeddings = HuggingFaceEmbeddings(
+        model_name="sentence-transformers/all-MiniLM-L6-v2"
+    )
+
+    vectorstore = FAISS.from_documents(texts, embeddings)
+
+    # 5️⃣ Save FAISS (runtime only)
+    vectorstore.save_local(FAISS_DIR)
+
+    return vectorstore