HuggingFace0920 commited on
Commit
de78368
·
verified ·
1 Parent(s): 7d9e249

Upload sync_data.sh

Browse files
Files changed (1) hide show
  1. sync_data.sh +17 -1
sync_data.sh CHANGED
@@ -120,8 +120,24 @@ sync_data() {
120
  python hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
121
 
122
  # 合并历史提交
123
- echo "正在合并历史提交..."
 
 
 
 
 
124
  python hf_sync.py super_squash "${HF_TOKEN}" "${DATASET_ID}"
 
 
 
 
 
 
 
 
 
 
 
125
 
126
  # 清理临时文件
127
  rm -f "/tmp/${backup_file}"
 
120
  python hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
121
 
122
  # 合并历史提交
123
+ SQUASH_FLAG_FILE="/tmp/last_squash_time"
124
+ NOW=$(date +%s)
125
+ SEVEN_DAYS=$((7*24*60*60))
126
+ if [ ! -f "$SQUASH_FLAG_FILE" ]; then
127
+ echo $NOW > "$SQUASH_FLAG_FILE"
128
+ echo "首次合并历史提交..."
129
  python hf_sync.py super_squash "${HF_TOKEN}" "${DATASET_ID}"
130
+ else
131
+ LAST=$(cat "$SQUASH_FLAG_FILE")
132
+ DIFF=$((NOW - LAST))
133
+ if [ $DIFF -ge $SEVEN_DAYS ]; then
134
+ echo $NOW > "$SQUASH_FLAG_FILE"
135
+ echo "距离上次合并已超过7天,正在合并历史提交..."
136
+ python hf_sync.py super_squash "${HF_TOKEN}" "${DATASET_ID}"
137
+ else
138
+ echo "距离上次合并未满7天,本次跳过合并历史提交。"
139
+ fi
140
+ fi
141
 
142
  # 清理临时文件
143
  rm -f "/tmp/${backup_file}"