Update app.py

app.py

@@ -17,6 +17,7 @@ from langchain_core.output_parsers import StrOutputParser
 from langchain_community.document_loaders import PyMuPDFLoader
 from langchain_openai import OpenAIEmbeddings
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_text_splitters import SpacyTextSplitter
 from langchain_core.prompts import ChatPromptTemplate
 from langchain_openai import ChatOpenAI
 import re
@@ -73,8 +74,8 @@ def summarize_pdf(pdf_file_path, num_clusters=10):
     docs = loader.load()
     full_text = "\n".join(doc.page_content for doc in docs)
     cleaned_full_text = clean_text(remove_references(full_text))
-
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=0, separators=["\n\n", "\n", ".", " "])
+    text_splitter = SpacyTextSplitter(chunk_size=500)
+    #text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=0, separators=["\n\n", "\n", ".", " "])
     split_contents = text_splitter.split_text(cleaned_full_text)
     embeddings = embeddings_model.embed_documents(split_contents)
 
@@ -103,8 +104,9 @@ def qa_pdf(pdf_file_path, query, num_clusters=5, similarity_threshold=0.6):
     docs = loader.load()
     full_text = "\n".join(doc.page_content for doc in docs)
     cleaned_full_text = clean_text(remove_references(full_text))
+    text_splitter = SpacyTextSplitter(chunk_size=500)
 
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=0, separators=["\n\n", "\n", ".", " "])
+    #text_splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=0, separators=["\n\n", "\n", ".", " "])
     split_contents = text_splitter.split_text(cleaned_full_text)
     embeddings = embeddings_model.embed_documents(split_contents)