Recursive character text splitter

app.py

@@ -12,6 +12,7 @@ from aimakerspace.vectordatabase import VectorDatabase
 from aimakerspace.openai_utils.chatmodel import ChatOpenAI
 import chainlit as cl
 from PyPDF2 import PdfReader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 
 system_template = """\
 Use the following context to answer a users question. If you cannot find the answer in the context, say you don't know the answer."""
@@ -48,7 +49,7 @@ class RetrievalAugmentedQAPipeline:
 
         return {"response": generate_response(), "context": context_list}
 
-text_splitter = CharacterTextSplitter()
+text_splitter = RecursiveCharacterTextSplitter()
 
 
 def process_text_file(file: AskFileResponse):
@@ -69,8 +70,8 @@ def process_text_file(file: AskFileResponse):
         documents = pdf_loader.load()
     else:
         raise ValueError("Provide a .txt or .pdf file")
-    texts = text_splitter.split_texts(documents)
-    
+    # texts = text_splitter.split_texts(documents)
+    texts = [x.page_content for x in text_splitter.transform_documents(documents)]
     return texts
 
 

requirements.txt

@@ -3,4 +3,5 @@ chainlit==0.7.700
 openai
 PyPDF2
 langchain-community
-pypdf
+pypdf
+langchain-text-splitters