wwforonce commited on
Commit
c3f29db
·
1 Parent(s): 51d370f

fix dataset archives

Browse files
Files changed (1) hide show
  1. sync_storage.py +3 -1
sync_storage.py CHANGED
@@ -378,12 +378,14 @@ This repository is automatically managed by the Open WebUI sync system.
378
  metadata = self._get_metadata()
379
 
380
  logger.info(f"Found {len(current_files)} files to preserve.")
 
 
381
 
382
  # 2. Download all current files to a temporary directory
383
  with tempfile.TemporaryDirectory() as tmpdir:
384
  logger.info(f"Step 2/5: Downloading current files to a temporary location...")
385
  for file_path in current_files:
386
- if file_path.startswith("data-") and (file_path not in metadata["archives"]):
387
  logger.info(f"Skipping {file_path} (not in archive)")
388
  continue
389
  self.api.hf_hub_download(
 
378
  metadata = self._get_metadata()
379
 
380
  logger.info(f"Found {len(current_files)} files to preserve.")
381
+ archive_filenames = [i["filename"] for i in metadata["archives"]]
382
+ logger.info(f"Archive filenames: {archive_filenames}")
383
 
384
  # 2. Download all current files to a temporary directory
385
  with tempfile.TemporaryDirectory() as tmpdir:
386
  logger.info(f"Step 2/5: Downloading current files to a temporary location...")
387
  for file_path in current_files:
388
+ if file_path.startswith("data-") and (file_path not in archive_filenames):
389
  logger.info(f"Skipping {file_path} (not in archive)")
390
  continue
391
  self.api.hf_hub_download(