import os import sys import re import tarfile from datetime import datetime, timezone from huggingface_hub import HfApi, hf_hub_download api = HfApi() repo_id = os.getenv("HF_DATASET") token = os.getenv("HF_TOKEN") OPENCLAW_DIR = "/root/.openclaw" REPO_TYPE = "dataset" # 备份文件命名:backup-YYYYmmdd-HHMMSS.tar.gz(带时间戳,便于轮转) BACKUP_PREFIX = "backup-" BACKUP_SUFFIX = ".tar.gz" BACKUP_RE = re.compile(r"^backup-\d{8}-\d{6}\.tar\.gz$") LEGACY_FILENAME = "latest_backup.tar.gz" # 兼容历史上的单文件备份 KEEP = 5 # 仅保留最近 5 份备份,多余的删除 # 备份时跳过的临时文件/目录 EXCLUDE_SUFFIXES = (".log", ".tmp", ".bak", ".pid") EXCLUDE_NAMES = {"__pycache__", "node_modules", ".git"} def _check_env(): if not repo_id or not token: print("Skip: HF_DATASET or HF_TOKEN not set") return False return True def should_exclude(tarinfo): """过滤掉不需要备份的临时文件""" name = os.path.basename(tarinfo.name) if name in EXCLUDE_NAMES: return None if any(name.endswith(s) for s in EXCLUDE_SUFFIXES): return None return tarinfo def _list_backups(): """返回 (备份文件名列表[新->旧], 仓库全部文件列表)""" files = api.list_repo_files(repo_id=repo_id, repo_type=REPO_TYPE, token=token) backups = sorted((f for f in files if BACKUP_RE.match(f)), reverse=True) return backups, files def restore(): if not _check_env(): return try: backups, files = _list_backups() # 优先恢复最新的时间戳备份;否则回退到旧的单文件备份 target = backups[0] if backups else ( LEGACY_FILENAME if LEGACY_FILENAME in files else None) if not target: print("Restore Note: no backup found in repo") return print(f"Downloading {target} from {repo_id}...") path = hf_hub_download( repo_id=repo_id, filename=target, repo_type=REPO_TYPE, token=token) with tarfile.open(path, "r:gz") as tar: tar.extractall(path=OPENCLAW_DIR) print(f"Success: restored from {target}") return True except Exception as e: print(f"Restore Note: no existing backup found or error: {e}") def _prune_and_squash(): """删除超过 KEEP 份的旧备份 + 清理遗留单文件,并压扁 git 历史真正回收存储。""" try: backups, files = _list_backups() # 1) 需要删除的:超过 KEEP 份的旧备份 + 遗留的 latest_backup.tar.gz to_delete = list(backups[KEEP:]) if LEGACY_FILENAME in files and len(backups) >= 1: to_delete.append(LEGACY_FILENAME) for name in to_delete: try: api.delete_file( path_in_repo=name, repo_id=repo_id, repo_type=REPO_TYPE, token=token, commit_message=f"prune old backup {name}") print(f"Pruned: {name}") except Exception as e: print(f"Prune warn ({name}): {e}") # 2) 关键一步:压扁全部历史,否则被删/被覆盖的旧版本 blob 仍占配额 try: api.super_squash_history( repo_id=repo_id, repo_type=REPO_TYPE, token=token, commit_message="squash backup history to reclaim storage") print("Squashed repo history (storage reclaimed)") except Exception as e: print(f"Squash warn: {e}") except Exception as e: print(f"Prune/squash warn: {e}") def backup(): if not _check_env(): return if not os.path.exists(OPENCLAW_DIR): print(f"Skip Backup: {OPENCLAW_DIR} does not exist") return ts = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") fname = f"{BACKUP_PREFIX}{ts}{BACKUP_SUFFIX}" uploaded = False try: with tarfile.open(fname, "w:gz") as tar: tar.add(OPENCLAW_DIR, arcname=".", filter=should_exclude) size_mb = os.path.getsize(fname) / (1024 * 1024) print(f"Backup archive size: {size_mb:.1f} MB ({fname})") api.upload_file( path_or_fileobj=fname, path_in_repo=fname, repo_id=repo_id, repo_type=REPO_TYPE, token=token) uploaded = True print(f"Backup {fname} uploaded.") except Exception as e: # 上传失败(如配额超限)必须显式告警,绝不能静默吞掉 print(f"!!! BACKUP FAILED (check HF storage quota / token!): {e}") finally: try: os.remove(fname) except OSError: pass # 仅在成功上传后再做轮转 + 压扁历史 if uploaded: _prune_and_squash() if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == "backup": backup() else: restore()