Update app.py

app.py

@@ -22,6 +22,8 @@ from langchain_community.chat_message_histories import ChatMessageHistory
 if not os.path.isdir('database'):
     os.system("unzip database.zip")
 
+clean_up_tokenization_spaces = True
+
 loader = DirectoryLoader('./database', glob="./*.txt", loader_cls=TextLoader)
 
 documents = loader.load()
@@ -29,10 +31,22 @@ documents = loader.load()
 text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
 texts = text_splitter.split_documents(documents)
 
+print()
+print("-------")
+print("TextSplitter, DirectoryLoader")
+print("-------")
+print("--")
+
 persist_directory = 'db'
 
 embedding = HuggingFaceEmbeddings()
 
+print()
+print("-------")
+print("Embeddings")
+print("-------")
+print("--")
+
 vectordb = Chroma.from_documents(documents=texts,
                                  embedding=embedding,
                                  persist_directory=persist_directory)
@@ -40,9 +54,21 @@ vectordb = Chroma.from_documents(documents=texts,
 vectordb.persist()
 vectordb = None
 
+print()
+print("-------")
+print("Chroma1")
+print("-------")
+print("--")
+
 vectordb = Chroma(persist_directory=persist_directory,
                   embedding_function=embedding)
 
+print()
+print("-------")
+print("Chroma2")
+print("-------")
+print("--")
+
 def format_docs(docs):
     return "\n\n".join(doc.page_content for doc in docs)
 
@@ -56,6 +82,12 @@ rag_chain = (
     | StrOutputParser()
 )
 
+print()
+print("-------")
+print("Retriever, Prompt, LLM, Rag_Chain")
+print("-------")
+print("--")
+
 contextualize_q_system_prompt = """Given a chat history and the latest user question \
 which might reference context in the chat history, formulate a standalone question \
 which can be understood without the chat history. Do NOT answer the question, \