Upload DocsLoader.py

utils/DocsLoader.py

@@ -0,0 +1,50 @@
+import os
+import tempfile
+import requests
+
+from fastapi import HTTPException
+from langchain_community.document_loaders import PyPDFLoader, Docx2txtLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.schema import Document
+
+
+def load_and_chunk(url: str) -> list[Document]:
+    resp = requests.get(url)
+    if resp.status_code != 200:
+        raise HTTPException(400, "Could not download document")
+
+    content_type = resp.headers.get("Content-Type", "").lower()
+    url_lower = url.lower()
+
+    if "application/pdf" in content_type or ".pdf" in url_lower:
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp:
+            tmp.write(resp.content)
+            tmp_path = tmp.name
+        try:
+            loader = PyPDFLoader(tmp_path)
+            docs = loader.load_and_split()
+        finally:
+            os.remove(tmp_path)
+
+    elif (
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document" in content_type
+        or ".docx" in url_lower
+    ):
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".docx") as tmp:
+            tmp.write(resp.content)
+            tmp_path = tmp.name
+        try:
+            loader = Docx2txtLoader(tmp_path)
+            docs = loader.load_and_split()
+        finally:
+            os.remove(tmp_path)
+
+    elif "text/plain" in content_type or ".txt" in url_lower:
+        text = resp.content.decode("utf-8", errors="ignore")
+        docs = [Document(page_content=text)]
+
+    else:
+        raise HTTPException(400, f"Unsupported document type: {content_type}")
+
+    splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=80)
+    return splitter.split_documents(docs)