Update utils/DocsLoader.py

utils/DocsLoader.py

@@ -1,21 +1,20 @@
 import os
 import tempfile
 import requests
-
+import re
 from fastapi import HTTPException
-# from langchain_community.document_loaders import PyPDFLoader, Docx2txtLoader
 from langchain_community.document_loaders import PyMuPDFLoader, Docx2txtLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.schema import Document
 
-
 def load_and_chunk(url: str) -> list[Document]:
     resp = requests.get(url)
     if resp.status_code != 200:
-        raise HTTPException(400, "Could not download document")
+        raise HTTPException(400, "Document download failed")
 
     content_type = resp.headers.get("Content-Type", "").lower()
     url_lower = url.lower()
+    text = ""
 
     if "application/pdf" in content_type or ".pdf" in url_lower:
         with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp:
@@ -23,29 +22,37 @@ def load_and_chunk(url: str) -> list[Document]:
             tmp_path = tmp.name
         try:
             loader = PyMuPDFLoader(tmp_path)
-            docs = loader.load_and_split()
+            # Extract full text first for better chunking
+            pages = loader.load()
+            text = "\n".join([p.page_content for p in pages])
         finally:
             os.remove(tmp_path)
 
-    elif (
-        "application/vnd.openxmlformats-officedocument.wordprocessingml.document" in content_type
-        or ".docx" in url_lower
-    ):
+    elif ("application/vnd.openxmlformats-officedocument.wordprocessingml.document" in content_type
+          or ".docx" in url_lower):
         with tempfile.NamedTemporaryFile(delete=False, suffix=".docx") as tmp:
             tmp.write(resp.content)
             tmp_path = tmp.name
         try:
             loader = Docx2txtLoader(tmp_path)
-            docs = loader.load_and_split()
+            docs = loader.load()
+            text = "\n".join([d.page_content for d in docs])
         finally:
             os.remove(tmp_path)
 
     elif "text/plain" in content_type or ".txt" in url_lower:
         text = resp.content.decode("utf-8", errors="ignore")
-        docs = [Document(page_content=text)]
 
     else:
         raise HTTPException(400, f"Unsupported document type: {content_type}")
 
-    splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=100)
-    return splitter.split_documents(docs)
+    # Clean and normalize text
+    text = re.sub(r'\s+', ' ', text).strip()
+    
+    # Semantic chunking
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1000,
+        chunk_overlap=150,
+        separators=["\n\n", "\n", ". ", "! ", "? ", " ", ""]
+    )
+    return splitter.split_text(text)