Spaces:
Running
Running
Download bucket_sync.py from tkgray/tom-hermes: direct link, hf CLI and curl.
- Browser
- Download file 7.26 kB
-
https://huggingface.co/spaces/tkgray/tom-hermes/resolve/main/bucket_sync.py
- Command line
-
hf download hf://spaces/tkgray/tom-hermes/bucket_sync.py
-
curl -L -o bucket_sync.py https://huggingface.co/spaces/tkgray/tom-hermes/resolve/main/bucket_sync.py
7.26 kB
| #!/opt/hermes/.venv/bin/python | |
| """Snapshot /opt/data to a HF Storage Bucket and restore it on a fresh boot. | |
| Why this exists: a bucket mounted *at* /opt/data cannot host Hermes — the gateway | |
| binds a Unix socket at $HERMES_HOME/gateway.sock and hf-mount (FUSE) refuses | |
| AF_UNIX nodes, so the gateway crash-loops ("gateway session bootstrap control | |
| listener unavailable"). Buckets are reached over the API here instead, so the | |
| runtime filesystem stays a normal local disk. | |
| Resilience model: | |
| * one timestamped object per snapshot under snapshots/, never overwritten, so a | |
| corrupt snapshot cannot destroy the last good one | |
| * each archive is verified (opens as gzip, non-empty, config.yaml present) | |
| BEFORE it is uploaded | |
| * latest.txt ships in the SAME commit as the archive, so a reader can never see | |
| a pointer to an object that failed to upload | |
| * restore walks newest-first and falls back to an older snapshot when the | |
| newest one does not yield a usable home directory | |
| """ | |
| import os | |
| import shutil | |
| import sqlite3 | |
| import sys | |
| import tarfile | |
| import tempfile | |
| import time | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from huggingface_hub import HfApi | |
| BUCKET = os.environ.get("HERMES_BUCKET", "tkgray/hermes-data") | |
| SNAP_DIR = "snapshots" | |
| SNAP_PREFIX = f"{SNAP_DIR}/hermes-data-" | |
| POINTER = "latest.txt" | |
| LEGACY_OBJECT = "hermes-data.tgz" # pre-rotation single-object layout | |
| RETAIN = int(os.environ.get("SNAPSHOT_RETAIN", "24")) | |
| HOME = Path(os.environ.get("HERMES_HOME", "/opt/data")) | |
| # Regenerable / runtime-only: never worth archiving, and .local is a whole | |
| # uv-managed CPython (~5k files) that would dominate every snapshot. | |
| SKIP_DIRS = { | |
| ".local", "cache", ".cache", "logs", "installs", "backups", | |
| "node_modules", "__pycache__", "tmp", ".git", "gateway", | |
| } | |
| SKIP_SUFFIX = (".sock", ".lock", ".db-wal", ".db-shm", ".db-journal", ".pid") | |
| def log(msg: str) -> None: | |
| print(f"[bucket] {msg}", flush=True) | |
| def _ignore(dirpath, names): | |
| return [n for n in names if n in SKIP_DIRS or n.endswith(SKIP_SUFFIX)] | |
| def _copy_db(src: Path, dst: Path) -> None: | |
| """Replace a possibly-torn file copy with a consistent sqlite backup.""" | |
| try: | |
| with sqlite3.connect(f"file:{src}?mode=ro", uri=True, timeout=5) as s: | |
| with sqlite3.connect(dst) as d: | |
| s.backup(d) | |
| except Exception as e: # locked / not-a-db: keep the plain copy | |
| log(f"db backup degraded for {src.name}: {e}") | |
| def _snapshot_names(api) -> list[str]: | |
| try: | |
| tree = api.list_bucket_tree(BUCKET, recursive=True) | |
| except TypeError: | |
| tree = api.list_bucket_tree(BUCKET) | |
| names = [] | |
| for node in tree: | |
| path = getattr(node, "path", None) | |
| if path is None and isinstance(node, dict): | |
| path = node.get("path") | |
| if path is None: | |
| path = str(node) | |
| if path.startswith(SNAP_PREFIX) and path.endswith(".tgz"): | |
| names.append(path) | |
| return sorted(names) | |
| def _verify_archive(path: Path) -> int: | |
| """Fail loudly rather than upload a snapshot we cannot read back.""" | |
| with tarfile.open(path, "r:gz") as tar: | |
| members = tar.getmembers() | |
| if not members: | |
| raise ValueError("archive is empty") | |
| if (HOME / "config.yaml").exists() and not any( | |
| m.name.rstrip("/").endswith("config.yaml") for m in members | |
| ): | |
| raise ValueError("archive is missing config.yaml") | |
| return len(members) | |
| def backup() -> int: | |
| api = HfApi() | |
| stage = Path(tempfile.mkdtemp(prefix="hermes-snap-")) | |
| try: | |
| target = stage / "data" | |
| shutil.copytree(HOME, target, symlinks=True, ignore=_ignore, dirs_exist_ok=True) | |
| for db in target.rglob("*.db"): | |
| src = HOME / db.relative_to(target) | |
| if src.is_file(): | |
| _copy_db(src, db) | |
| archive = stage / "snap.tgz" | |
| with tarfile.open(archive, "w:gz") as tar: | |
| tar.add(target, arcname=".") | |
| entries = _verify_archive(archive) | |
| size = archive.stat().st_size | |
| name = f"{SNAP_PREFIX}{datetime.now(timezone.utc):%Y%m%d-%H%M%S}.tgz" | |
| # Archive + pointer in one commit: the pointer can never reference an | |
| # object that did not make it. | |
| api.batch_bucket_files(BUCKET, add=[(archive, name), (name.encode(), POINTER)]) | |
| log(f"backup ok: {size / 1e6:.1f} MB, {entries} entries, verified -> {name}") | |
| names = _snapshot_names(api) | |
| stale = names[:-RETAIN] if len(names) > RETAIN else [] | |
| if stale: | |
| api.batch_bucket_files(BUCKET, delete=stale) | |
| log(f"pruned {len(stale)} old snapshot(s); keeping newest {RETAIN}") | |
| return 0 | |
| finally: | |
| shutil.rmtree(stage, ignore_errors=True) | |
| def _candidate_names(api) -> list[str]: | |
| """Pointer first, then every other snapshot newest-first, then the legacy object.""" | |
| ordered: list[str] = [] | |
| try: | |
| with tempfile.TemporaryDirectory() as tmp: | |
| local = Path(tmp) / "pointer" | |
| api.download_bucket_files(BUCKET, [(POINTER, local)], raise_on_missing_files=True) | |
| pointed = local.read_text(encoding="utf-8-sig").strip() | |
| if pointed: | |
| ordered.append(pointed) | |
| except Exception as e: | |
| log(f"no pointer ({type(e).__name__}); falling back to newest snapshot") | |
| for name in reversed(_snapshot_names(api)): | |
| if name not in ordered: | |
| ordered.append(name) | |
| # Pre-rotation layout: one object at the bucket root. | |
| if LEGACY_OBJECT not in ordered: | |
| ordered.append(LEGACY_OBJECT) | |
| return ordered | |
| def restore() -> int: | |
| if (HOME / "config.yaml").exists(): | |
| log("restore skipped: /opt/data already populated") | |
| return 0 | |
| api = HfApi() | |
| for name in _candidate_names(api): | |
| workdir = Path(tempfile.mkdtemp(prefix="hermes-restore-")) | |
| local = workdir / "snap.tgz" | |
| try: | |
| api.download_bucket_files(BUCKET, [(name, local)], raise_on_missing_files=True) | |
| with tarfile.open(local, "r:gz") as tar: | |
| tar.extractall(HOME) | |
| if (HOME / "config.yaml").exists(): | |
| log(f"restore ok: {name} -> {len(list(HOME.iterdir()))} entries in /opt/data") | |
| return 0 | |
| log(f"restored {name} but no config.yaml; trying an older snapshot") | |
| except Exception as e: | |
| log(f"restore of {name} failed: {type(e).__name__}: {e}") | |
| finally: | |
| shutil.rmtree(workdir, ignore_errors=True) | |
| log("no usable snapshot; starting fresh") | |
| return 0 | |
| def main() -> int: | |
| cmd = sys.argv[1] if len(sys.argv) > 1 else "backup" | |
| if cmd == "restore": | |
| return restore() | |
| if cmd == "backup": | |
| return backup() | |
| if cmd == "loop": | |
| interval = int(os.environ.get("SYNC_INTERVAL", "300")) | |
| log(f"loop started: every {interval}s -> {BUCKET}/{SNAP_PREFIX}*.tgz (retain {RETAIN})") | |
| while True: | |
| time.sleep(interval) | |
| try: | |
| backup() | |
| except Exception as e: | |
| log(f"backup failed: {type(e).__name__}: {e}") | |
| print(f"unknown command: {cmd}", file=sys.stderr) | |
| return 2 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |