Update sync_data.sh

sync_data.sh

@@ -31,9 +31,9 @@ import os
 print(f'HF_TOKEN is set: {os.environ.get(\"HF_TOKEN\") is not None}')
 print(f'DATASET_ID is set: {os.environ.get(\"DATASET_ID\") is not None}')
 
-def manage_backups(api, repo_id, max_files=50):
+def manage_backups(api, repo_id_val, max_files=50):
     print('Managing old backups...')
-    files = api.list_repo_files(repo_id=repo_id, repo_type='dataset')
+    files = api.list_repo_files(repo_id=repo_id_val, repo_type='dataset')
     backup_files = [f for f in files if f.startswith('$BACKUP_PREFIX') and f.endswith('.tar.gz')]
     backup_files.sort()
 
@@ -43,7 +43,7 @@ def manage_backups(api, repo_id, max_files=50):
         for file_to_delete in files_to_delete:
             try:
                 print(f'Deleting old backup: {file_to_delete}')
-                api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id, repo_type='dataset')
+                api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id_val, repo_type='dataset')
                 print(f'Successfully deleted: {file_to_delete}')
             except Exception as e:
                 print(f'Error deleting {file_to_delete}: {str(e)}')
@@ -52,16 +52,17 @@ def manage_backups(api, repo_id, max_files=50):
 
 api = HfApi(token='$token')
 try:
-    print(f'Uploading file: $file_path to {repo_id} as $file_name')
+    repo_id_val = os.environ.get('DATASET_ID') # 从环境变量中获取 repo_id
+    print(f'Uploading file: $file_path to {repo_id_val} as $file_name')
     api.upload_file(
         path_or_fileobj='$file_path',
         path_in_repo='$file_name',
-        repo_id=f'{repo_id}', # 使用 f-string 传递 repo_id
+        repo_id=repo_id_val,
         repo_type='dataset'
     )
     print(f'Successfully uploaded $file_name')
 
-    manage_backups(api, f'{repo_id}') # 使用 f-string 传递 repo_id
+    manage_backups(api, repo_id_val)
 except Exception as e:
     print(f'Error uploading file: {str(e)}')
 "
@@ -86,8 +87,9 @@ print(f'DATASET_ID is set: {os.environ.get(\"DATASET_ID\") is not None}')
 
 api = HfApi(token='$token')
 try:
-    print(f'Listing files in Dataset: {repo_id}')
-    files = api.list_repo_files(repo_id=f'{repo_id}', repo_type='dataset') # 使用 f-string 传递 repo_id
+    repo_id_val = os.environ.get('DATASET_ID') # 从环境变量中获取 repo_id
+    print(f'Listing files in Dataset: {repo_id_val}')
+    files = api.list_repo_files(repo_id=repo_id_val, repo_type='dataset')
     backup_files = [f for f in files if f.startswith('$BACKUP_PREFIX') and f.endswith('.tar.gz')]
 
     if not backup_files:
@@ -99,7 +101,7 @@ try:
 
     with tempfile.TemporaryDirectory() as temp_dir:
         filepath = api.hf_hub_download(
-            repo_id=f'{repo_id}', # 使用 f-string 传递 repo_id
+            repo_id=repo_id_val,
             filename=latest_backup,
             repo_type='dataset',
             local_dir=temp_dir