#!/bin/sh # 检查环境变量 if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then echo "Starting without backup functionality - missing HF_TOKEN or DATASET_ID" exit 1 fi # 激活虚拟环境 . /home/app/venv/bin/activate # 上传备份 cat > /home/app/uptime-kuma/hf_sync.py << 'EOL' from huggingface_hub import HfApi import sys import os import tarfile import tempfile import requests import json def manage_backups(api, repo_id, max_files=50): files = api.list_repo_files(repo_id=repo_id, repo_type="dataset") backup_files = [f for f in files if f.startswith('backup_') and f.endswith('.tar.gz')] backup_files.sort() if len(backup_files) >= max_files: files_to_delete = backup_files[:(len(backup_files) - max_files + 1)] for file_to_delete in files_to_delete: try: api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id, repo_type="dataset") print(f'Deleted old backup: {file_to_delete}') except Exception as e: print(f'Error deleting {file_to_delete}: {str(e)}') def upload_backup(file_path, file_name, token, repo_id): api = HfApi(token=token) try: api.upload_file( path_or_fileobj=file_path, path_in_repo=file_name, repo_id=repo_id, repo_type="dataset" ) print(f"Successfully uploaded {file_name}") manage_backups(api, repo_id) except Exception as e: print(f"Error uploading file: {str(e)}") # 下载最新备份 def download_latest_backup(token, repo_id): try: api = HfApi(token=token) files = api.list_repo_files(repo_id=repo_id, repo_type="dataset") backup_files = [f for f in files if f.startswith('backup_') and f.endswith('.tar.gz')] if not backup_files: print("No backup files found") return latest_backup = sorted(backup_files)[-1] with tempfile.TemporaryDirectory() as temp_dir: filepath = api.hf_hub_download( repo_id=repo_id, filename=latest_backup, repo_type="dataset", local_dir=temp_dir ) if filepath and os.path.exists(filepath): with tarfile.open(filepath, 'r:gz') as tar: tar.extractall('/home/app/uptime-kuma/') print(f"Successfully restored backup from {latest_backup}") except Exception as e: print(f"Error downloading backup: {str(e)}") def clear_tmp_bak(account, password, repo_ids, save_num): # 登录获取 token login_url = "https://huggingface.co/login" login_data = { "username": account, "password": password } try: # 发送登录请求,允许重定向 session = requests.Session() login_response = session.post(login_url, data=login_data, allow_redirects=True) if not login_response.ok: print("Login failed") return # 从 session 的 cookies 中获取 token token = session.cookies.get('token') if not token: print("Failed to get token") return # 准备请求头 headers = { 'Cookie': f'token={token}', 'Content-Type': 'application/json' } # 处理多个仓库ID repo_id_list = [repo_id.strip() for repo_id in repo_ids.split(',')] print(f"Processing {len(repo_id_list)} repositories for cleanup") for repo_id in repo_id_list: print(f"\nProcessing repository: {repo_id}") # 首先获取 LFS 文件列表 list_url = f"https://huggingface.co/api/datasets/{repo_id}/lfs-files" list_response = requests.get(list_url, headers=headers) if not list_response.ok: print(f"Failed to get file list for {repo_id}: {list_response.status_code}") continue # 解析文件列表并按时间排序 files = list_response.json() files.sort(key=lambda x: x['pushedAt'], reverse=True) # Print current file list print(f"Current LFS file list for {repo_id}:") for file in files: print(f"Filename: {file['filename']}, Upload time: {file['pushedAt']}, FileOid: {file['fileOid']}") # 保留最新的文件,删除其他文件 if len(files) > int(save_num): # 跳过要保留的文件 files_to_delete = files[int(save_num):] sha_list = [file['fileOid'] for file in files_to_delete] lfs_url = f"https://huggingface.co/api/datasets/{repo_id}/lfs-files/batch" # 构造删除请求的数据 delete_data = { "deletions": { "rewriteHistory": True, "sha": sha_list } } # 发送删除请求 delete_response = requests.post(lfs_url, headers=headers, data=json.dumps(delete_data)) if delete_response.ok: print(f"Successfully deleted {len(sha_list)} old files from {repo_id}") else: print(f"Cleanup failed for {repo_id}: {delete_response.status_code} - {delete_response.text}") else: print(f"No files need to be cleaned for {repo_id}") except Exception as e: print(f"Error during cleanup process: {str(e)}") if __name__ == "__main__": action = sys.argv[1] if action == "upload": token = os.environ.get("HF_TOKEN") repo_id = os.environ.get("DATASET_ID") file_path = sys.argv[2] file_name = sys.argv[3] upload_backup(file_path, file_name, token, repo_id) elif action == "download": token = os.environ.get("HF_TOKEN") repo_id = os.environ.get("DATASET_ID") download_latest_backup(token, repo_id) elif action == "clear": token = os.environ.get("HF_TOKEN") account = os.environ.get("ACCOUNT") password = os.environ.get("PASSWORD") repo_ids = os.environ.get("CLEAR_DATA_IDS") save_num = os.environ.get("SAVENUMS", "10") clear_tmp_bak(account, password, repo_ids, save_num) EOL # 首次启动时从HuggingFace下载最新备份 echo "Downloading latest backup from HuggingFace..." python hf_sync.py download # 同步函数 sync_data() { while true; do echo "Starting sync process at $(date)" # 确保数据目录存在 if [ -d "/home/app/uptime-kuma/data" ]; then # 创建备份 cd /home/app/uptime-kuma timestamp=$(date +%Y%m%d_%H%M%S) backup_file="backup_${timestamp}.tar.gz" # 压缩数据目录 tar -czf "/tmp/${backup_file}" data/ # 上传到HuggingFace echo "Uploading backup to HuggingFace..." python hf_sync.py upload "/tmp/${backup_file}" "${backup_file}" # 清理临时文件 rm -f "/tmp/${backup_file}" else echo "Data directory does not exist yet, waiting for next sync..." fi # 同步间隔 SYNC_INTERVAL=${SYNC_INTERVAL:-7200} current_timestamp=$(date +%s) next_timestamp=$((current_timestamp + SYNC_INTERVAL)) next_time=$(date -r $next_timestamp "+%Y-%m-%d %H:%M:%S" 2>/dev/null || date -d "@$next_timestamp" "+%Y-%m-%d %H:%M:%S" 2>/dev/null || echo "$(date) + $SYNC_INTERVAL seconds") echo "Next sync at ${next_time} (in ${SYNC_INTERVAL} seconds)..." sleep $SYNC_INTERVAL done } # 同步函数 clear_lfs() { # Check if required environment variables exist if [ -z "$ACCOUNT" ] || [ -z "$PASSWORD" ] || [ -z "$CLEAR_DATA_IDS" ]; then echo "Skipping clear process - missing ACCOUNT, PASSWORD or CLEAR_DATA_IDS environment variables" return fi SAVENUMS=${SAVENUMS:-10} while true; do echo "Starting clear process at $(date)" # 直接调用清理脚本,从环境变量读取参数 python hf_sync.py clear # 同步间隔 默认7天清理一次 CLEAR_INTERVAL=${CLEAR_INTERVAL:-604800} current_timestamp=$(date +%s) next_timestamp=$((current_timestamp + CLEAR_INTERVAL)) next_time=$(date -r $next_timestamp "+%Y-%m-%d %H:%M:%S" 2>/dev/null || date -d "@$next_timestamp" "+%Y-%m-%d %H:%M:%S" 2>/dev/null || echo "$(date) + $CLEAR_INTERVAL seconds") echo "Next clear at ${next_time} (in ${CLEAR_INTERVAL} seconds)..." sleep $CLEAR_INTERVAL done } # 启动同步进程 sync_data & # 启动清理进程 clear_lfs & # 启动主应用 exec node server/server.js