update splitter

app.py

@@ -60,6 +60,7 @@ from langchain.document_loaders import (
     UnstructuredExcelLoader
 )
 from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.text_splitter import TokenTextSplitter
 from langchain.docstore.document import Document
 import langchain
 import asyncio
@@ -355,7 +356,8 @@ def process_documents_3(ignored_files: List[str] = []) -> List[Document]:
         print("No new documents to load")
         exit(0)
     print(f"Loaded {len(documents)} new documents from {source_directory}")
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=8000, chunk_overlap=1000)
+    # text_splitter = RecursiveCharacterTextSplitter(chunk_size=8000, chunk_overlap=1000)
+    text_splitter = TokenTextSplitter(chunk_size=4000, chunk_overlap=500)
     texts = text_splitter.split_documents(documents)
     print(f"Split into {len(texts)} chunks of text (max. {chunk_size} tokens each)")
     return texts