Upload sync_data.sh

sync_data.sh

@@ -1,27 +1,28 @@
 #!/bin/sh
 
-# 检查必须的环境变量是否已设置
+# 检查环境变量
 if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
-    echo "HF_TOKEN 和 DATASET_ID 环境变量未设置"
+    echo "未检测到 HF_TOKEN 或 DATASET_ID，备份功能不可用"
     exit 1
 fi
 
-# 激活 Python 虚拟环境
+# 激活虚拟环境
 . /app/venv/bin/activate
 
-# 生成 HuggingFace 同步脚本（Python 实现核心同步逻辑）
+# 生成同步脚本
 cat > hf_sync.py << 'EOL'
+# HuggingFace 同步脚本
 from huggingface_hub import HfApi
 import sys
 import os
 import tarfile
 import tempfile
 
-# 管理备份文件，超过最大数量则自动删除最旧的备份
+# 管理备份文件数量，超出最大数量则自动删除最旧的备份
 def manage_backups(api, repo_id, max_files=50):
     files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
     backup_files = [f for f in files if f.startswith('backup_') and f.endswith('.tar.gz')]
-    backup_files.sort()  # 从旧到新
+    backup_files.sort()
     if len(backup_files) >= max_files:
         files_to_delete = backup_files[:(len(backup_files) - max_files + 1)]
         for file_to_delete in files_to_delete:
@@ -41,12 +42,13 @@ def upload_backup(file_path, file_name, token, repo_id):
             repo_id=repo_id,
             repo_type="dataset"
         )
-        print(f"成功上传备份: {file_name}")
+        print(f"成功上传 {file_name}")
         manage_backups(api, repo_id)
     except Exception as e:
         print(f"上传文件出错: {str(e)}")
 
-# 下载最新的备份并解压到指定目录
+
+# 下载最新备份
 def download_latest_backup(token, repo_id, extract_path):
     try:
         api = HfApi(token=token)
@@ -66,95 +68,73 @@ def download_latest_backup(token, repo_id, extract_path):
             if filepath and os.path.exists(filepath):
                 with tarfile.open(filepath, 'r:gz') as tar:
                     tar.extractall(extract_path)
-                print(f"成功恢复备份: {latest_backup}")
+                print(f"已成功恢复备份: {latest_backup}")
     except Exception as e:
         print(f"下载备份出错: {str(e)}")
 
-# 清理所有 LFS（大文件存储）备份文件，只保留最新的 keep_latest 个
-def clear_lfs_files(token, repo_id, keep_latest=3):
-    try:
-        api = HfApi(token=token)
-        files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
-        lfs_files = [f for f in files if f.endswith('.tar.gz')]
-        lfs_files.sort()  # 从旧到新
-        files_to_delete = lfs_files[:-keep_latest] if len(lfs_files) > keep_latest else []
-        if not files_to_delete:
-            print("没有需要删除的旧 LFS 文件")
-            return
-        for lfs_file in files_to_delete:
-            try:
-                api.delete_file(path_in_repo=lfs_file, repo_id=repo_id, repo_type="dataset")
-                print(f"已删除旧 LFS 文件: {lfs_file}")
-            except Exception as e:
-                print(f"删除 LFS 文件出错: {str(e)}")
-    except Exception as e:
-        print(f"清理 LFS 文件出错: {str(e)}")
-
-# 合并所有历史提交为一个（Super Squash），可选保留最新的几个提交
-def super_squash_history(token, repo_id, keep_latest=1):
+# 合并历史提交，仅保留最近的若干次提交
+def super_squash_history(token, repo_id, keep_latest=3):
     try:
         api = HfApi(token=token)
         api.super_squash_history(repo_id=repo_id, repo_type="dataset", keep_last_commits=keep_latest)
-        print(f"成功执行 super_squash_history，将历史合并，仅保留最新的 {keep_latest} 个提交。")
+        print(f"历史合并完成，仅保留最近 {keep_latest} 次提交。")
     except Exception as e:
-        print(f"super_squash_history 执行出错: {str(e)}")
+        print(f"合并历史出错: {str(e)}")
 
-# 主函数，根据命令行参数执行不同操作
+# 主函数
 if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print("用法: python hf_sync.py <操作> [参数]")
-        sys.exit(1)
     action = sys.argv[1]
-    token = os.getenv('HF_TOKEN')
-    repo_id = os.getenv('DATASET_ID')
+    token = sys.argv[2]
+    repo_id = sys.argv[3]
     if action == "upload":
-        if len(sys.argv) < 4:
-            print("用法: python hf_sync.py upload <file_path> <file_name>")
-            sys.exit(1)
-        file_path = sys.argv[2]
-        file_name = sys.argv[3]
+        file_path = sys.argv[4]
+        file_name = sys.argv[5]
         upload_backup(file_path, file_name, token, repo_id)
     elif action == "download":
-        extract_path = sys.argv[2] if len(sys.argv) > 2 else '.'
+        extract_path = sys.argv[4] if len(sys.argv) > 4 else '.'  # 默认为当前目录
         download_latest_backup(token, repo_id, extract_path)
-    elif action == "clear_lfs":
-        keep_latest = int(sys.argv[2]) if len(sys.argv) > 2 else 3
-        clear_lfs_files(token, repo_id, keep_latest)
     elif action == "super_squash":
-        keep_latest = int(sys.argv[2]) if len(sys.argv) > 2 else 1
+        keep_latest = int(sys.argv[4]) if len(sys.argv) > 4 else 3
         super_squash_history(token, repo_id, keep_latest)
-    else:
-        print("无效操作。可用操作: upload, download, clear_lfs, super_squash")
-        sys.exit(1)
 EOL
-
-# 首次启动时自动下载最新备份
+# 首次启动时从 HuggingFace 下载最新备份（解压到应用目录）
 echo "正在从 HuggingFace 下载最新备份..."
-python hf_sync.py download "./"
+python hf_sync.py download "${HF_TOKEN}" "${DATASET_ID}" "./"  
 
-# 定时同步函数
+# 同步函数
 sync_data() {
     while true; do
-        echo "开始同步流程，当前时间: $(date)"
+        echo "同步进程启动于 $(date)"
+        
+        # 确保数据目录存在（请根据实际路径修改）
         STORAGE_PATH="./storage"
         if [ -d "${STORAGE_PATH}" ]; then
-            BACKUP_FILE="backup_$(date +%Y%m%d_%H%M%S).tar.gz"
-            tar -czf "${BACKUP_FILE}" -C "${STORAGE_PATH}" .
-            echo "已创建备份: ${BACKUP_FILE}"
-            python hf_sync.py upload "${BACKUP_FILE}" "${BACKUP_FILE}"
-            rm -f "${BACKUP_FILE}"
-            python hf_sync.py clear_lfs 3
+            # 创建备份
+            timestamp=$(date +%Y%m%d_%H%M%S)
+            backup_file="backup_${timestamp}.tar.gz"
+            
+            # 压缩目录（使用-C避免包含父路径）
+            tar -czf "/tmp/${backup_file}" -C "$(dirname "${STORAGE_PATH}")" "$(basename "${STORAGE_PATH}")"
+            
+            # 上传到 HuggingFace
+            echo "正在上传备份到 HuggingFace..."
+            python hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
+            
+            # 清理临时文件
+            rm -f "/tmp/${backup_file}"
         else
-            echo "数据目录不存在: ${STORAGE_PATH}"
+            echo "存储目录 ${STORAGE_PATH} 不存在，等待中..."
         fi
+        
+        # 同步间隔
         SYNC_INTERVAL=${SYNC_INTERVAL:-7200}
-        echo "等待 ${SYNC_INTERVAL} 秒后进行下一次同步..."
+        echo "下次同步将在 ${SYNC_INTERVAL} 秒后进行..."
         sleep $SYNC_INTERVAL
     done
 }
 
-# 后台启动同步进程
+# 启动同步进程
 sync_data &
 
-# 启动主应用（请根据实际路径调整）
+# 启动主应用（请根据实际路径修改）
 exec bash install_reader.sh