Update utils/DocsLoader.py

utils/DocsLoader.py

@@ -6,8 +6,10 @@ from fastapi import HTTPException
 # from langchain_community.document_loaders import PyPDFLoader, Docx2txtLoader
 from langchain_community.document_loaders import PyMuPDFLoader, Docx2txtLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
+from  langchain_text_splitters.sentence_transformers import SentenceTransformersTokenTextSplitter 
 from langchain.schema import Document
 
+MODEL_DIR = os.path.join("/tmp", "e5-large-v2")
 
 def load_and_chunk(url: str) -> list[Document]:
     print(url)
@@ -48,5 +50,6 @@ def load_and_chunk(url: str) -> list[Document]:
     else:
         raise HTTPException(400, f"Unsupported document type: {content_type}")
 
-    splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100,separators=["\n\n", "\n", ".", " ", ""])
+    # splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100,separators=["\n\n", "\n", ".", " ", ""])
+    splitter = SentenceTransformersTokenTextSplitter(model_name = MODEL_DIR,tokens_per_chunk=500, chunk_overlap=80)
     return splitter.split_documents(docs)