Create functions.py

functions.py

@@ -0,0 +1,42 @@
+def create_retriever_from_chroma(vectorstore_path="./docs/chroma/", search_type='mmr', k=7, chunk_size=300, chunk_overlap=30,lambda_mult= 0.7):
+    
+    model_name = "Alibaba-NLP/gte-large-en-v1.5"
+    model_kwargs = {'device': 'cpu',
+                   "trust_remote_code" : 'False'}
+    encode_kwargs = {'normalize_embeddings': True}
+    embeddings = HuggingFaceEmbeddings(
+        model_name=model_name,
+        model_kwargs=model_kwargs,
+        encode_kwargs=encode_kwargs
+    )
+
+    
+
+    if os.path.exists(vectorstore_path) and os.listdir(vectorstore_path):
+        vectorstore = Chroma(persist_directory=vectorstore_path,embedding_function=embeddings)
+        
+    else:
+        st.write("Vector store doesnt exist and will be created now")
+        loader = DirectoryLoader('./data/', glob="./*.txt", loader_cls=TextLoader)
+        docs = loader.load()
+        
+        
+        text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
+        chunk_size=chunk_size, chunk_overlap=chunk_overlap,
+        separators=["\n\n \n\n","\n\n\n", "\n\n", r"In \[[0-9]+\]", r"\n+", r"\s+"],
+        is_separator_regex = True
+    )
+        split_docs = text_splitter.split_documents(docs)
+
+        
+        vectorstore = Chroma.from_documents(
+            documents=split_docs, embedding=embeddings, persist_directory=vectorstore_path
+        )
+        
+    
+    retriever=vectorstore.as_retriever(search_type = search_type, search_kwargs={"k": k})
+
+    
+    
+
+    return retriever