Update app.py

app.py

@@ -4,23 +4,35 @@ from langchain.vectorstores import FAISS
 from langchain.chains import ConversationalRetrievalChain
 from langchain.llms import HuggingFacePipeline
 from langchain.memory import ConversationBufferMemory
+from langchain_community.retrievers import BM25Retriever
+from langchain.retrievers import EnsembleRetriever
 import pandas as pd
 df = pd.read_csv('NLP.csv')
-corpus = df['text']
+corpus = df['text'][:300]
 #Chunking
 splitter = RecursiveCharacterTextSplitter(chunk_size=200,chunk_overlap = 10)
 texts = sum([splitter.split_text(doc) for doc in corpus], [])
 # Embeddings
-embeddings = HuggingFaceEmbeddings(model_name='all-MiniLM-L6-v2')
+embeddings = HuggingFaceEmbeddings(model_name='all-mnpnet-base-v2')
 # Indexing
-db = FAISS.from_texts(texts[:300],embeddings)
-retriever = db.as_retriever(search_kwargs={'k':2})
+db = FAISS.from_texts(texts,embeddings)
+retriever = db.as_retriever(search_kwargs={'k':5})
+
+# BM25
+bm25 = BM25Retriever.from_texts(texts)
+bm25.k =5
+
+# Hy_brid retriever
+hybrid_retriever = EnsembleRetriever(
+    retrievers = [retriever,bm25],
+    weights = [0.7,0.3]
+)
 # Model
 llm = HuggingFacePipeline.from_model_id(model_id='google/flan-t5-large',task='text2text-generation')
 # Memory
 memory = ConversationBufferMemory(memory_key='chat_history',return_messages=True)
 # Combine previous steps
-qa = ConversationalRetrievalChain.from_llm(llm=llm,retriever=retriever,memory=memory)
+qa = ConversationalRetrievalChain.from_llm(llm=llm,retriever=hybrid_retriever,memory=memory)
 def ans_ques(ques):
     result = qa({'question':ques})
     return result['answer']