Update utils/DocsLoader.py

utils/DocsLoader.py

@@ -1,20 +1,21 @@
 import os
 import tempfile
 import requests
-import re
+
 from fastapi import HTTPException
+# from langchain_community.document_loaders import PyPDFLoader, Docx2txtLoader
 from langchain_community.document_loaders import PyMuPDFLoader, Docx2txtLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.schema import Document
 
+
 def load_and_chunk(url: str) -> list[Document]:
     resp = requests.get(url)
     if resp.status_code != 200:
-        raise HTTPException(400, "Document download failed")
+        raise HTTPException(400, "Could not download document")
 
     content_type = resp.headers.get("Content-Type", "").lower()
     url_lower = url.lower()
-    text = ""
 
     if "application/pdf" in content_type or ".pdf" in url_lower:
         with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp:
@@ -22,37 +23,29 @@ def load_and_chunk(url: str) -> list[Document]:
             tmp_path = tmp.name
         try:
             loader = PyMuPDFLoader(tmp_path)
-            # Extract full text first for better chunking
-            pages = loader.load()
-            text = "\n".join([p.page_content for p in pages])
+            docs = loader.load_and_split()
         finally:
             os.remove(tmp_path)
 
-    elif ("application/vnd.openxmlformats-officedocument.wordprocessingml.document" in content_type
-          or ".docx" in url_lower):
+    elif (
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document" in content_type
+        or ".docx" in url_lower
+    ):
         with tempfile.NamedTemporaryFile(delete=False, suffix=".docx") as tmp:
             tmp.write(resp.content)
             tmp_path = tmp.name
         try:
             loader = Docx2txtLoader(tmp_path)
-            docs = loader.load()
-            text = "\n".join([d.page_content for d in docs])
+            docs = loader.load_and_split()
         finally:
             os.remove(tmp_path)
 
     elif "text/plain" in content_type or ".txt" in url_lower:
         text = resp.content.decode("utf-8", errors="ignore")
+        docs = [Document(page_content=text)]
 
     else:
         raise HTTPException(400, f"Unsupported document type: {content_type}")
 
-    # Clean and normalize text
-    text = re.sub(r'\s+', ' ', text).strip()
-    
-    # Semantic chunking
-    splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000,
-        chunk_overlap=150,
-        separators=["\n\n", "\n", ". ", "! ", "? ", " ", ""]
-    )
-    return splitter.split_text(text)
+    splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=150)
+    return splitter.split_documents(docs)