HuggingFace0920 commited on
Commit
a53e76c
·
verified ·
1 Parent(s): c5c4b27

Upload sync_data.sh

Browse files
Files changed (1) hide show
  1. sync_data.sh +6 -6
sync_data.sh CHANGED
@@ -72,12 +72,12 @@ def download_latest_backup(token, repo_id, extract_path):
72
  except Exception as e:
73
  print(f"下载备份出错: {str(e)}")
74
 
75
- # 合并历史提交,仅保留最近的若干次提交
76
- def super_squash_history(token, repo_id, keep_latest=3):
77
  try:
78
  api = HfApi(token=token)
79
- api.super_squash_history(repo_id=repo_id, repo_type="dataset", keep_last=keep_latest)
80
- print(f"历史合并完成,仅保留最近 {keep_latest} 次提交。")
81
  except Exception as e:
82
  print(f"合并历史出错: {str(e)}")
83
 
@@ -120,8 +120,8 @@ sync_data() {
120
  echo "正在上传备份到 HuggingFace..."
121
  python hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
122
 
123
- # 定期合并历史提交,只保留最新的3个提交
124
- echo "正在合并历史提交,仅保留最近3次提交..."
125
  python hf_sync.py super_squash "${HF_TOKEN}" "${DATASET_ID}" 3
126
 
127
  # 清理临时文件
 
72
  except Exception as e:
73
  print(f"下载备份出错: {str(e)}")
74
 
75
+ # 合并历史提交
76
+ def super_squash_history(token, repo_id):
77
  try:
78
  api = HfApi(token=token)
79
+ api.super_squash_history(repo_id=repo_id, repo_type="dataset")
80
+ print("历史合并完成。")
81
  except Exception as e:
82
  print(f"合并历史出错: {str(e)}")
83
 
 
120
  echo "正在上传备份到 HuggingFace..."
121
  python hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
122
 
123
+ # 合并历史提交
124
+ echo "正在合并历史提交..."
125
  python hf_sync.py super_squash "${HF_TOKEN}" "${DATASET_ID}" 3
126
 
127
  # 清理临时文件