Update utils/DocsLoader.py

utils/DocsLoader.py

@@ -72,8 +72,29 @@ os.makedirs(CACHE_DIR, exist_ok=True)
 def load_and_chunk(url: str) -> list[Document]:
     print(f"[Loader] URL: {url}")
 
-    # Use hash of the URL to avoid download if cached
-    content_hash = hashlib.md5(url.encode()).hexdigest()
+    # Try to get content length
+    try:
+        head = requests.head(url)
+        file_size = int(head.headers.get("Content-Length", 0))
+    except:
+        file_size = 0
+
+    USE_STREAM = file_size > 30 * 1024 * 1024  # >30MB
+
+    # Download content (streamed if large)
+    content = b""
+    hasher = hashlib.md5()
+
+    with requests.get(url, stream=USE_STREAM) as resp:
+        if resp.status_code != 200:
+            raise HTTPException(400, "Could not download document")
+
+        for chunk in resp.iter_content(chunk_size=8192):
+            hasher.update(chunk)
+            content += chunk
+
+    # Cache key based on content hash
+    content_hash = hasher.hexdigest()
     cache_file = os.path.join(CACHE_DIR, f"{content_hash}.pkl")
 
     # Return cached version if exists
@@ -82,17 +103,6 @@ def load_and_chunk(url: str) -> list[Document]:
         with open(cache_file, "rb") as f:
             return pickle.load(f)
 
-    # Download content
-    head = requests.head(url)
-    file_size = int(head.headers.get("Content-Length", 0))
-    USE_STREAM = file_size > 30 * 1024 * 1024  # > 30MB
-
-    resp = requests.get(url, stream=USE_STREAM)
-    if resp.status_code != 200:
-        raise HTTPException(400, "Could not download document")
-
-    content = resp.content
-
     # Determine content type
     content_type = resp.headers.get("Content-Type", "").lower()
     url_lower = url.lower()
@@ -135,5 +145,3 @@ def load_and_chunk(url: str) -> list[Document]:
     print(f"💾 Chunks cached to {cache_file}")
 
     return chunks
-
-