Fix: add repo_type=dataset and force_download=True for public dataset

chatbot_retriever.py

@@ -72,7 +72,7 @@ import os
 DATASET_REPO = "07Codex07/PrepGraph-Data"
 
 def ensure_data_dir():
-    """Make sure data/ folder exists and contains downloaded PDFs from HF dataset."""
+    """Ensure data/ folder exists and download PDFs from the Hugging Face dataset with correct subfolders."""
     data_dir = os.getenv("DATA_DIR", "data")
     os.makedirs(data_dir, exist_ok=True)
 
@@ -98,19 +98,23 @@ def ensure_data_dir():
 
     local_paths = []
     for f in files:
-        local_path = os.path.join(data_dir, f.replace("/", "_"))
-        if not os.path.exists(local_path):
+        # ✅ Keep the original folder structure (e.g., data/pyqs/...)
+        download_path = os.path.join(data_dir, f)
+        os.makedirs(os.path.dirname(download_path), exist_ok=True)
+
+        if not os.path.exists(download_path):
             print(f"📥 Downloading {f} from Hugging Face (public dataset)...")
             downloaded = hf_hub_download(
                 repo_id=DATASET_REPO,
                 filename=f,
-                repo_type="dataset",       # ✅ tells HF it's a dataset
-                force_download=True        # ✅ bypass any bad cached 401s
+                repo_type="dataset",        # ✅ specify dataset type
+                force_download=True         # ✅ force refresh cache if needed
             )
-            os.rename(downloaded, local_path)
-        local_paths.append(local_path)
-    return local_paths
+            os.replace(downloaded, download_path)
 
+        local_paths.append(download_path)
+
+    return local_paths
 
 
 def detect_subject(fname: str) -> Optional[str]: