Spaces:
Paused
Paused
fix dataset archives
Browse files- sync_storage.py +3 -1
sync_storage.py
CHANGED
|
@@ -378,12 +378,14 @@ This repository is automatically managed by the Open WebUI sync system.
|
|
| 378 |
metadata = self._get_metadata()
|
| 379 |
|
| 380 |
logger.info(f"Found {len(current_files)} files to preserve.")
|
|
|
|
|
|
|
| 381 |
|
| 382 |
# 2. Download all current files to a temporary directory
|
| 383 |
with tempfile.TemporaryDirectory() as tmpdir:
|
| 384 |
logger.info(f"Step 2/5: Downloading current files to a temporary location...")
|
| 385 |
for file_path in current_files:
|
| 386 |
-
if file_path.startswith("data-") and (file_path not in
|
| 387 |
logger.info(f"Skipping {file_path} (not in archive)")
|
| 388 |
continue
|
| 389 |
self.api.hf_hub_download(
|
|
|
|
| 378 |
metadata = self._get_metadata()
|
| 379 |
|
| 380 |
logger.info(f"Found {len(current_files)} files to preserve.")
|
| 381 |
+
archive_filenames = [i["filename"] for i in metadata["archives"]]
|
| 382 |
+
logger.info(f"Archive filenames: {archive_filenames}")
|
| 383 |
|
| 384 |
# 2. Download all current files to a temporary directory
|
| 385 |
with tempfile.TemporaryDirectory() as tmpdir:
|
| 386 |
logger.info(f"Step 2/5: Downloading current files to a temporary location...")
|
| 387 |
for file_path in current_files:
|
| 388 |
+
if file_path.startswith("data-") and (file_path not in archive_filenames):
|
| 389 |
logger.info(f"Skipping {file_path} (not in archive)")
|
| 390 |
continue
|
| 391 |
self.api.hf_hub_download(
|