new version of rag

app.py

@@ -95,20 +95,18 @@ def get_embedding_model():
     return HuggingFaceEmbedding(model_name="sentence-transformers/all-MiniLM-L6-v2")
 
 def list_zip_files_in_repo(repo_id: str) -> List[str]:
-    files = list_repo_files(repo_id, token=HF_TOKEN)
+    files = list_repo_files(repo_id, repo_type="dataset", token=HF_TOKEN)  # Add repo_type="dataset"
     return [f for f in files if f.startswith(JSON_FILES_DIR) and f.endswith('.zip')]
 
 def download_file_from_hf(repo_id: str, path_in_repo: str, dest_dir: str) -> str:
-    local_path = hf_hub_download(repo_id=repo_id, filename=path_in_repo, repo_type="dataset", token=HF_TOKEN)
-    base = os.path.basename(local_path)
-    dst = os.path.join(dest_dir, base)
-    if local_path != dst:
-        try:
-            with open(local_path, 'rb') as r, open(dst, 'wb') as w:
-                w.write(r.read())
-        except Exception:
-            pass
-    return dst
+    local_path = hf_hub_download(
+        repo_id=repo_id, 
+        filename=path_in_repo, 
+        repo_type="dataset", 
+        token=HF_TOKEN,
+        local_dir=dest_dir  # Add this to download directly to dest_dir
+    )
+    return local_path  # Return the path directly
 
 def read_jsons_from_zip(zip_path: str) -> List[Dict[str, Any]]:
     docs = []