Update utils/DocsLoader.py

utils/DocsLoader.py

@@ -66,8 +66,8 @@ def load_and_chunk(url: str) -> list[Document]:
     #             sentence_docs.append(Document(page_content=sent, metadata=doc.metadata))
     full_text = "\n".join([doc.page_content for doc in docs])  
 
-    splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=150)
-    # splitter = SentenceTransformersTokenTextSplitter(model_name = MODEL_DIR,tokens_per_chunk=500, chunk_overlap=80)
+    # splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=150)
+    splitter = SentenceTransformersTokenTextSplitter(model_name = MODEL_DIR,tokens_per_chunk=512, chunk_overlap=80)
     # return splitter.split_documents(docs)
     # return splitter.split_documents([Document(page_content=full_text)])
     return splitter.create_documents([full_text])