Update utils/DocsLoader.py

utils/DocsLoader.py

@@ -50,6 +50,6 @@ def load_and_chunk(url: str) -> list[Document]:
     else:
         raise HTTPException(400, f"Unsupported document type: {content_type}")
 
-    splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=120,separators=["\n\n", "\n", ".", " ", ""])
-    # splitter = SentenceTransformersTokenTextSplitter(model_name = MODEL_DIR,tokens_per_chunk=500, chunk_overlap=80)
+    # splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=120,separators=["\n\n", "\n", ".", " ", ""])
+    splitter = SentenceTransformersTokenTextSplitter(model_name = MODEL_DIR,tokens_per_chunk=500, chunk_overlap=80)
     return splitter.split_documents(docs)