wwforonce commited on
Commit
8bf0297
·
1 Parent(s): 8f048eb

fix download

Browse files
Files changed (1) hide show
  1. sync_storage.py +12 -6
sync_storage.py CHANGED
@@ -225,18 +225,24 @@ This repository is automatically managed by the Open WebUI sync system.
225
  try:
226
  metadata = self._get_metadata()
227
  current_files = list_repo_files(self.repo_id, repo_type="dataset", token=self.token)
 
 
228
 
229
  logger.info(f"Found {len(current_files)} files to preserve.")
230
  archive_filenames = [i["filename"] for i in metadata["archives"]]
231
  logger.info(f"Archive filenames: {archive_filenames}")
232
  logger.info(f"Current filenames: {current_files}")
 
233
  latest_link = self.latest_link
234
- if (latest_link not in current_files) and (len(archive_filenames) > 0):
235
- latest_link = archive_filenames[0]
236
- logger.info(f"Latest link not found, falling back to first archive: {latest_link}")
237
- else:
238
- logger.error("No archives found in repository")
239
- return
 
 
 
240
  logger.info(f"Downloading latest data archive: {latest_link}")
241
  # First try the latest link
242
  file_path = self.api.hf_hub_download(
 
225
  try:
226
  metadata = self._get_metadata()
227
  current_files = list_repo_files(self.repo_id, repo_type="dataset", token=self.token)
228
+ current_files = [ i for i in current_files if i.startswith("data-") ]
229
+ current_files = sorted(current_files)
230
 
231
  logger.info(f"Found {len(current_files)} files to preserve.")
232
  archive_filenames = [i["filename"] for i in metadata["archives"]]
233
  logger.info(f"Archive filenames: {archive_filenames}")
234
  logger.info(f"Current filenames: {current_files}")
235
+
236
  latest_link = self.latest_link
237
+ logger.info(f"Current latest_link: {latest_link}")
238
+
239
+ if (latest_link not in current_files):
240
+ if (len(current_files) > 0):
241
+ latest_link = current_files[-1]
242
+ logger.info(f"Latest link not found, falling back to last current file: {latest_link}")
243
+ else:
244
+ logger.error("No archives found in repository")
245
+ return
246
  logger.info(f"Downloading latest data archive: {latest_link}")
247
  # First try the latest link
248
  file_path = self.api.hf_hub_download(