Update sync_data.sh

sync_data.sh

@@ -92,15 +92,10 @@ python3 -c "
 from huggingface_hub import HfApi
 try:
     api = HfApi(token='$HF_TOKEN')
-    
-    # 创建本地测试文件
     with open('$TEMP_DIR/test_file', 'w') as f:
         f.write('test')
-    
-    # 上传测试文件
     test_file_name = '$TEST_FILE_NAME'
     print(f'正在上传测试文件: {test_file_name}')
-    
     api.upload_file(
         path_or_fileobj='$TEMP_DIR/test_file',
         path_in_repo=test_file_name,
@@ -108,8 +103,6 @@ try:
         repo_type='dataset'
     )
     print('成功上传测试文件到Dataset')
-    
-    # 删除已上传的测试文件
     print('正在删除测试文件...')
     api.delete_file(
         path_in_repo=test_file_name,
@@ -117,7 +110,6 @@ try:
         repo_type='dataset'
     )
     print('已成功删除测试文件')
-    
 except Exception as e:
     print(f'Dataset权限测试失败: {str(e)}')
     exit(1)
@@ -135,14 +127,11 @@ rm -f "$TEMP_DIR/test_file"
 upload_backup() {
     file_path="$1"
     file_name="$2"
-    
     if [ ! -f "$file_path" ]; then
         log_error "备份文件不存在: $file_path"
         return 1
     fi
-    
     log_info "开始上传备份: $file_name ($(du -h $file_path | cut -f1))"
-    
     python3 -c "
 from huggingface_hub import HfApi
 import sys
@@ -153,7 +142,6 @@ def manage_backups(api, repo_id, max_files=10):
         files = api.list_repo_files(repo_id=repo_id, repo_type='dataset')
         backup_files = [f for f in files if f.startswith('sillytavern_backup_') and f.endswith('.tar.gz')]
         backup_files.sort()
-        
         if len(backup_files) >= max_files:
             files_to_delete = backup_files[:(len(backup_files) - max_files + 1)]
             for file_to_delete in files_to_delete:
@@ -168,33 +156,24 @@ token='$HF_TOKEN'
 repo_id='$DATASET_ID'
 try:
     api = HfApi(token=token)
-    
-    # 检查文件大小
     file_size = os.path.getsize('$file_path')
     print(f'备份文件大小: {file_size / (1024*1024):.2f} MB')
-    # 确认Dataset存在
     try:
         dataset_info = api.dataset_info(repo_id=repo_id)
         print(f'Dataset信息: {dataset_info.id}')
     except Exception as e:
         print(f'获取Dataset信息失败: {str(e)}')
-    
     start_time = time.time()
     print(f'开始上传: {start_time}')
-    
-    # 上传文件
     api.upload_file(
         path_or_fileobj='$file_path',
         path_in_repo='$file_name',
         repo_id=repo_id,
         repo_type='dataset'
     )
-    
     end_time = time.time()
     print(f'上传完成，耗时: {end_time - start_time:.2f} 秒')
     print(f'成功上传 $file_name')
-    
-    # 管理备份
     manage_backups(api, repo_id)
 except Exception as e:
     print(f'上传文件时出错: {str(e)}')
@@ -212,7 +191,6 @@ except Exception as e:
 # 下载最新备份
 download_latest_backup() {
     log_info "开始下载最新备份..."
-    
     python3 -c "
 from huggingface_hub import HfApi
 import sys
@@ -223,33 +201,23 @@ import time
 try:
     api = HfApi(token='$HF_TOKEN')
     print('已创建API实例')
-    
-    # 列出仓库文件
     try:
         files = api.list_repo_files(repo_id='$DATASET_ID', repo_type='dataset')
         print(f'仓库文件数量: {len(files)}')
     except Exception as e:
         print(f'列出仓库文件失败: {str(e)}')
         sys.exit(1)
-    
     backup_files = [f for f in files if f.startswith('sillytavern_backup_') and f.endswith('.tar.gz')]
     print(f'找到备份文件数量: {len(backup_files)}')
-    
     if not backup_files:
         print('未找到备份文件')
         sys.exit(0)
-    
-    # 按名称排序（实际上是按时间戳排序）
     latest_backup = sorted(backup_files)[-1]
     print(f'最新备份文件: {latest_backup}')
-    
     with tempfile.TemporaryDirectory() as temp_dir:
         print(f'创建临时目录: {temp_dir}')
-        
         start_time = time.time()
         print(f'开始下载: {start_time}')
-        
-        # 下载文件
         try:
             filepath = api.hf_hub_download(
                 repo_id='$DATASET_ID',
@@ -261,18 +229,11 @@ try:
         except Exception as e:
             print(f'下载文件失败: {str(e)}')
             sys.exit(1)
-        
         end_time = time.time()
         print(f'下载完成，耗时: {end_time - start_time:.2f} 秒')
-        
         if filepath and os.path.exists(filepath):
-            # 确保目标目录存在
             os.makedirs('$DATA_DIR', exist_ok=True)
-            
-            # 检查文件权限
             print(f'文件权限: {oct(os.stat(filepath).st_mode)[-3:]}')
-            
-            # 解压文件
             try:
                 with tarfile.open(filepath, 'r:gz') as tar:
                     print('开始解压文件...')
@@ -281,7 +242,6 @@ try:
             except Exception as e:
                 print(f'解压文件失败: {str(e)}')
                 sys.exit(1)
-            
             print(f'成功从 {latest_backup} 恢复备份')
         else:
             print('下载的文件路��无效')
@@ -306,35 +266,63 @@ download_latest_backup
 # 同步函数
 sync_data() {
     log_info "数据同步服务已启动"
-    
     while true; do
         log_info "开始同步进程，时间: $(date)"
-        
         if [ -d "$DATA_DIR" ]; then
             timestamp=$(date +%Y%m%d_%H%M%S)
             backup_file="sillytavern_backup_${timestamp}.tar.gz"
             backup_path="${TEMP_DIR}/${backup_file}"
-            
             log_info "创建备份文件: $backup_path"
-            
-            # 检查数据目录内容
             file_count=$(find "$DATA_DIR" -type f | wc -l)
             log_info "数据目录文件数量: $file_count"
-            
             if [ "$file_count" -eq 0 ]; then
                 log_info "数据目录为空，跳过备份"
             else
-                # 压缩数据目录
                 tar -czf "$backup_path" -C "$DATA_DIR" .
                 if [ $? -ne 0 ]; then
                     log_error "创建压缩文件失败"
                 else
                     log_info "压缩文件创建成功: $(du -h $backup_path | cut -f1)"
-                    
                     # 上传备份
                     log_info "正在上传备份到HuggingFace..."
                     upload_backup "$backup_path" "$backup_file"
-                    
+
+                    # 每7天合并一次历史提交
+                    SQUASH_FLAG_FILE="/tmp/last_squash_time"
+                    NOW=$(date +%s)
+                    SEVEN_DAYS=$((7*24*60*60))
+                    if [ ! -f "$SQUASH_FLAG_FILE" ]; then
+                        echo $NOW > "$SQUASH_FLAG_FILE"
+                        log_info "首次合并历史提交..."
+                        python3 -c "
+from huggingface_hub import HfApi
+try:
+    api = HfApi(token='$HF_TOKEN')
+    api.super_squash_history(repo_id='$DATASET_ID', repo_type='dataset')
+    print('历史合并完成。')
+except Exception as e:
+    print(f'合并历史出错: {str(e)}')
+"
+                    else
+                        LAST=$(cat "$SQUASH_FLAG_FILE")
+                        DIFF=$((NOW - LAST))
+                        if [ $DIFF -ge $SEVEN_DAYS ]; then
+                            echo $NOW > "$SQUASH_FLAG_FILE"
+                            log_info "距离上次合并已超过7天，正在合并历史提交..."
+                            python3 -c "
+from huggingface_hub import HfApi
+try:
+    api = HfApi(token='$HF_TOKEN')
+    api.super_squash_history(repo_id='$DATASET_ID', repo_type='dataset')
+    print('历史合并完成。')
+except Exception as e:
+    print(f'合并历史出错: {str(e)}')
+"
+                        else
+                            log_info "距离上次合并未满7天，本次跳过合并历史提交。"
+                        fi
+                    fi
+
                     # 删除临时备份文件
                     rm -f "$backup_path"
                     log_info "已删除临时备份文件"
@@ -345,7 +333,6 @@ sync_data() {
             mkdir -p "$DATA_DIR"
             chmod -R 777 "$DATA_DIR"
         fi
-        
         # 设置同步间隔
         SYNC_INTERVAL=${SYNC_INTERVAL:-3600}
         log_info "下次同步将在 ${SYNC_INTERVAL} 秒后进行..."
@@ -354,4 +341,4 @@ sync_data() {
 }
 
 # 启动同步进程
-sync_data 
+sync_data