Update utils/DocsLoader.py

utils/DocsLoader.py

@@ -3,12 +3,14 @@ import tempfile
 import requests
 
 from fastapi import HTTPException
+import nltk
 # from langchain_community.document_loaders import PyPDFLoader, Docx2txtLoader
 from langchain_community.document_loaders import PyMuPDFLoader, Docx2txtLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from  langchain_text_splitters.sentence_transformers import SentenceTransformersTokenTextSplitter  # give better results but slow can use later for project
 from langchain.schema import Document
-
+# Download NLTK sentence tokenizer
+nltk.download('punkt', quiet=True)
 MODEL_DIR = os.path.join("/tmp", "e5-large-v2")
 
 def load_and_chunk(url: str) -> list[Document]:
@@ -49,7 +51,15 @@ def load_and_chunk(url: str) -> list[Document]:
 
     else:
         raise HTTPException(400, f"Unsupported document type: {content_type}")
+    # --- Step 1: Sentence split ---
+    sentence_docs = []
+    for doc in docs:
+        sentences = nltk.sent_tokenize(doc.page_content)
+        for sent in sentences:
+            if sent.strip():
+                sentence_docs.append(Document(page_content=sent, metadata=doc.metadata))
 
-    # splitter = RecursiveCharacterTextSplitter(chunk_size=600, chunk_overlap=80,separators=["\n\n", "\n", ".", " ", ""])
-    splitter = SentenceTransformersTokenTextSplitter(model_name = MODEL_DIR,tokens_per_chunk=500, chunk_overlap=80)
-    return splitter.split_documents(docs)
+    splitter = RecursiveCharacterTextSplitter(chunk_size=600, chunk_overlap=80,separators=["\n\n", "\n", ".", " ", ""])
+    # splitter = SentenceTransformersTokenTextSplitter(model_name = MODEL_DIR,tokens_per_chunk=500, chunk_overlap=80)
+    # return splitter.split_documents(docs)
+    return splitter.split_documents(sentence_docs)