#!/opt/hermes/.venv/bin/python """Snapshot /opt/data to a HF Storage Bucket and restore it on a fresh boot. Why this exists: a bucket mounted *at* /opt/data cannot host Hermes — the gateway binds a Unix socket at $HERMES_HOME/gateway.sock and hf-mount (FUSE) refuses AF_UNIX nodes, so the gateway crash-loops ("gateway session bootstrap control listener unavailable"). Buckets are reached over the API here instead, so the runtime filesystem stays a normal local disk. Resilience model: * one timestamped object per snapshot under snapshots/, never overwritten, so a corrupt snapshot cannot destroy the last good one * each archive is verified (opens as gzip, non-empty, config.yaml present) BEFORE it is uploaded * latest.txt ships in the SAME commit as the archive, so a reader can never see a pointer to an object that failed to upload * restore walks newest-first and falls back to an older snapshot when the newest one does not yield a usable home directory """ import os import shutil import sqlite3 import sys import tarfile import tempfile import time from datetime import datetime, timezone from pathlib import Path from huggingface_hub import HfApi BUCKET = os.environ.get("HERMES_BUCKET", "tkgray/hermes-data") SNAP_DIR = "snapshots" SNAP_PREFIX = f"{SNAP_DIR}/hermes-data-" POINTER = "latest.txt" LEGACY_OBJECT = "hermes-data.tgz" # pre-rotation single-object layout RETAIN = int(os.environ.get("SNAPSHOT_RETAIN", "24")) HOME = Path(os.environ.get("HERMES_HOME", "/opt/data")) # Regenerable / runtime-only: never worth archiving, and .local is a whole # uv-managed CPython (~5k files) that would dominate every snapshot. SKIP_DIRS = { ".local", "cache", ".cache", "logs", "installs", "backups", "node_modules", "__pycache__", "tmp", ".git", "gateway", } SKIP_SUFFIX = (".sock", ".lock", ".db-wal", ".db-shm", ".db-journal", ".pid") def log(msg: str) -> None: print(f"[bucket] {msg}", flush=True) def _ignore(dirpath, names): return [n for n in names if n in SKIP_DIRS or n.endswith(SKIP_SUFFIX)] def _copy_db(src: Path, dst: Path) -> None: """Replace a possibly-torn file copy with a consistent sqlite backup.""" try: with sqlite3.connect(f"file:{src}?mode=ro", uri=True, timeout=5) as s: with sqlite3.connect(dst) as d: s.backup(d) except Exception as e: # locked / not-a-db: keep the plain copy log(f"db backup degraded for {src.name}: {e}") def _snapshot_names(api) -> list[str]: try: tree = api.list_bucket_tree(BUCKET, recursive=True) except TypeError: tree = api.list_bucket_tree(BUCKET) names = [] for node in tree: path = getattr(node, "path", None) if path is None and isinstance(node, dict): path = node.get("path") if path is None: path = str(node) if path.startswith(SNAP_PREFIX) and path.endswith(".tgz"): names.append(path) return sorted(names) def _verify_archive(path: Path) -> int: """Fail loudly rather than upload a snapshot we cannot read back.""" with tarfile.open(path, "r:gz") as tar: members = tar.getmembers() if not members: raise ValueError("archive is empty") if (HOME / "config.yaml").exists() and not any( m.name.rstrip("/").endswith("config.yaml") for m in members ): raise ValueError("archive is missing config.yaml") return len(members) def backup() -> int: api = HfApi() stage = Path(tempfile.mkdtemp(prefix="hermes-snap-")) try: target = stage / "data" shutil.copytree(HOME, target, symlinks=True, ignore=_ignore, dirs_exist_ok=True) for db in target.rglob("*.db"): src = HOME / db.relative_to(target) if src.is_file(): _copy_db(src, db) archive = stage / "snap.tgz" with tarfile.open(archive, "w:gz") as tar: tar.add(target, arcname=".") entries = _verify_archive(archive) size = archive.stat().st_size name = f"{SNAP_PREFIX}{datetime.now(timezone.utc):%Y%m%d-%H%M%S}.tgz" # Archive + pointer in one commit: the pointer can never reference an # object that did not make it. api.batch_bucket_files(BUCKET, add=[(archive, name), (name.encode(), POINTER)]) log(f"backup ok: {size / 1e6:.1f} MB, {entries} entries, verified -> {name}") names = _snapshot_names(api) stale = names[:-RETAIN] if len(names) > RETAIN else [] if stale: api.batch_bucket_files(BUCKET, delete=stale) log(f"pruned {len(stale)} old snapshot(s); keeping newest {RETAIN}") return 0 finally: shutil.rmtree(stage, ignore_errors=True) def _candidate_names(api) -> list[str]: """Pointer first, then every other snapshot newest-first, then the legacy object.""" ordered: list[str] = [] try: with tempfile.TemporaryDirectory() as tmp: local = Path(tmp) / "pointer" api.download_bucket_files(BUCKET, [(POINTER, local)], raise_on_missing_files=True) pointed = local.read_text(encoding="utf-8-sig").strip() if pointed: ordered.append(pointed) except Exception as e: log(f"no pointer ({type(e).__name__}); falling back to newest snapshot") for name in reversed(_snapshot_names(api)): if name not in ordered: ordered.append(name) # Pre-rotation layout: one object at the bucket root. if LEGACY_OBJECT not in ordered: ordered.append(LEGACY_OBJECT) return ordered def restore() -> int: if (HOME / "config.yaml").exists(): log("restore skipped: /opt/data already populated") return 0 api = HfApi() for name in _candidate_names(api): workdir = Path(tempfile.mkdtemp(prefix="hermes-restore-")) local = workdir / "snap.tgz" try: api.download_bucket_files(BUCKET, [(name, local)], raise_on_missing_files=True) with tarfile.open(local, "r:gz") as tar: tar.extractall(HOME) if (HOME / "config.yaml").exists(): log(f"restore ok: {name} -> {len(list(HOME.iterdir()))} entries in /opt/data") return 0 log(f"restored {name} but no config.yaml; trying an older snapshot") except Exception as e: log(f"restore of {name} failed: {type(e).__name__}: {e}") finally: shutil.rmtree(workdir, ignore_errors=True) log("no usable snapshot; starting fresh") return 0 def main() -> int: cmd = sys.argv[1] if len(sys.argv) > 1 else "backup" if cmd == "restore": return restore() if cmd == "backup": return backup() if cmd == "loop": interval = int(os.environ.get("SYNC_INTERVAL", "300")) log(f"loop started: every {interval}s -> {BUCKET}/{SNAP_PREFIX}*.tgz (retain {RETAIN})") while True: time.sleep(interval) try: backup() except Exception as e: log(f"backup failed: {type(e).__name__}: {e}") print(f"unknown command: {cmd}", file=sys.stderr) return 2 if __name__ == "__main__": sys.exit(main())