Update app.py

app.py

@@ -178,15 +178,37 @@ def getRAGChain(customerName,customerDistrict, custDetailsPresent,vectordb):
     )
     return chain
 
-def createVectorDB(documents):
+def createVectorDB(documents,embeddingModelID):
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=1500, chunk_overlap=150)
-    texts = text_splitter.split_documents(documents)
+    texts = []
+    for document in documents:
+        tokenized_input = tokenizer.tokenize(document.page_content)
+        print("Token Count::::::::::" + str(len(tokenized_input)))
+        if (len(tokenized_input) > 1000):
+            print("Splitting Content using RTS")
+            splitted_doc = text_splitter.split_documents([document])
+            texts.extend(splitted_doc)
+        #             for text in texts:
+        #                 print("splitted content:"+str(len(text.page_content)))
+        #                 print(text.page_content)
+        elif (len(tokenized_input) < 1000 and len(tokenized_input) > 1):
+            texts.append(document)
+    # texts = text_splitter.split_documents(documents)
     print("All chunk List START ***********************\n\n")
     pretty_print_docs(texts)
     print("All chunk List END ***********************\n\n")
-    embeddings = getEmbeddingModel(0)
-    vectordb = Chroma.from_documents(texts, embeddings)
+    embeddings = getEmbeddingModel(embeddingModelID)
+    print("Embedding Started >>>>>>>>>>>>>>>>>>", datetime.now().strftime("%H:%M:%S"))
+    vectordb = Chroma.from_documents(texts, embeddings, collection_metadata={"hnsw:space": "cosine"})
+    print("Vector Store Creation Completed*********************************\n\n")
     return vectordb
+    # texts = text_splitter.split_documents(documents)
+    # print("All chunk List START ***********************\n\n")
+    # pretty_print_docs(texts)
+    # print("All chunk List END ***********************\n\n")
+    # embeddings = getEmbeddingModel(0)
+    # vectordb = Chroma.from_documents(texts, embeddings)
+    # return vectordb
 
 def createPrompt(cName, cCity, custDetailsPresent):
     cProfile = "Customer's Name is " + cName + "\nCustomer's lives in or customer's Resident State or Customer's place is " + cCity + "\n"
@@ -213,7 +235,7 @@ def createPrompt(cName, cCity, custDetailsPresent):
     PROMPT = PromptTemplate(template=prompt_template, input_variables=["history", "context", "question"])
     return PROMPT
 
-vectordb = createVectorDB(loadKB(False, False, uploads_dir, None))
+vectordb = createVectorDB(loadKB(False, False, uploads_dir, None),defaultEmbeddingModelID)
 
 @app.route('/', methods=['GET'])
 def test():
@@ -287,7 +309,7 @@ def file_Upload():
         embeddingModelID = int(request.form.getlist('embeddingModelID')[0])
     global vectordb
     vectordb = createVectorDB(documents, embeddingModelID)
-    vectordb=createVectorDB(documents)
+    #vectordb=createVectorDB(documents)
     return render_template("index.html")
 
 if __name__ == '__main__':