Update utils/DocsLoader.py

utils/DocsLoader.py

@@ -66,13 +66,23 @@ from langchain.schema import Document
 from langchain_text_splitters.sentence_transformers import SentenceTransformersTokenTextSplitter  # better but slower
 
 MODEL_DIR = os.path.join("/tmp", "e5-large-v2")
-CACHE_DIR = "/tmp/chunk_data"
+CACHE_DIR = os.path.join(os.path.dirname(__file__), "../chunk_data")
 os.makedirs(CACHE_DIR, exist_ok=True)
 
 def load_and_chunk(url: str) -> list[Document]:
-    print(f"[Loader] Fetching: {url}")
+    print(f"[Loader] URL: {url}")
 
-    # Get content length to decide whether to stream
+    # Use hash of the URL to avoid download if cached
+    content_hash = hashlib.md5(url.encode()).hexdigest()
+    cache_file = os.path.join(CACHE_DIR, f"{content_hash}.pkl")
+
+    # Return cached version if exists
+    if os.path.exists(cache_file):
+        print(f"✅ Loaded chunks from disk cache (key={content_hash})")
+        with open(cache_file, "rb") as f:
+            return pickle.load(f)
+
+    # Download content
     head = requests.head(url)
     file_size = int(head.headers.get("Content-Length", 0))
     USE_STREAM = file_size > 30 * 1024 * 1024  # > 30MB
@@ -81,20 +91,7 @@ def load_and_chunk(url: str) -> list[Document]:
     if resp.status_code != 200:
         raise HTTPException(400, "Could not download document")
 
-    # Hash file content to create a unique cache key
-    hasher = hashlib.md5()
-    content = b""
-    for chunk in resp.iter_content(8192):
-        hasher.update(chunk)
-        content += chunk
-    content_hash = hasher.hexdigest()
-    cache_file = os.path.join(CACHE_DIR, f"{content_hash}.pkl")
-
-    # Return cached version if exists
-    if os.path.exists(cache_file):
-        print(f"✅ Loaded chunks from disk cache (key={content_hash})")
-        with open(cache_file, "rb") as f:
-            return pickle.load(f)
+    content = resp.content
 
     # Determine content type
     content_type = resp.headers.get("Content-Type", "").lower()
@@ -139,3 +136,4 @@ def load_and_chunk(url: str) -> list[Document]:
 
     return chunks
 
+