Spaces:
Paused
Paused
fix download
Browse files- sync_storage.py +12 -6
sync_storage.py
CHANGED
|
@@ -225,18 +225,24 @@ This repository is automatically managed by the Open WebUI sync system.
|
|
| 225 |
try:
|
| 226 |
metadata = self._get_metadata()
|
| 227 |
current_files = list_repo_files(self.repo_id, repo_type="dataset", token=self.token)
|
|
|
|
|
|
|
| 228 |
|
| 229 |
logger.info(f"Found {len(current_files)} files to preserve.")
|
| 230 |
archive_filenames = [i["filename"] for i in metadata["archives"]]
|
| 231 |
logger.info(f"Archive filenames: {archive_filenames}")
|
| 232 |
logger.info(f"Current filenames: {current_files}")
|
|
|
|
| 233 |
latest_link = self.latest_link
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
|
|
|
|
|
|
|
|
|
| 240 |
logger.info(f"Downloading latest data archive: {latest_link}")
|
| 241 |
# First try the latest link
|
| 242 |
file_path = self.api.hf_hub_download(
|
|
|
|
| 225 |
try:
|
| 226 |
metadata = self._get_metadata()
|
| 227 |
current_files = list_repo_files(self.repo_id, repo_type="dataset", token=self.token)
|
| 228 |
+
current_files = [ i for i in current_files if i.startswith("data-") ]
|
| 229 |
+
current_files = sorted(current_files)
|
| 230 |
|
| 231 |
logger.info(f"Found {len(current_files)} files to preserve.")
|
| 232 |
archive_filenames = [i["filename"] for i in metadata["archives"]]
|
| 233 |
logger.info(f"Archive filenames: {archive_filenames}")
|
| 234 |
logger.info(f"Current filenames: {current_files}")
|
| 235 |
+
|
| 236 |
latest_link = self.latest_link
|
| 237 |
+
logger.info(f"Current latest_link: {latest_link}")
|
| 238 |
+
|
| 239 |
+
if (latest_link not in current_files):
|
| 240 |
+
if (len(current_files) > 0):
|
| 241 |
+
latest_link = current_files[-1]
|
| 242 |
+
logger.info(f"Latest link not found, falling back to last current file: {latest_link}")
|
| 243 |
+
else:
|
| 244 |
+
logger.error("No archives found in repository")
|
| 245 |
+
return
|
| 246 |
logger.info(f"Downloading latest data archive: {latest_link}")
|
| 247 |
# First try the latest link
|
| 248 |
file_path = self.api.hf_hub_download(
|