tom-hermes / bucket_sync.py
tkgray's picture
restore: legacy single-object fallback
0f1863e verified
Raw History Blame Contribute Delete
7.26 kB
#!/opt/hermes/.venv/bin/python
"""Snapshot /opt/data to a HF Storage Bucket and restore it on a fresh boot.
Why this exists: a bucket mounted *at* /opt/data cannot host Hermes — the gateway
binds a Unix socket at $HERMES_HOME/gateway.sock and hf-mount (FUSE) refuses
AF_UNIX nodes, so the gateway crash-loops ("gateway session bootstrap control
listener unavailable"). Buckets are reached over the API here instead, so the
runtime filesystem stays a normal local disk.
Resilience model:
* one timestamped object per snapshot under snapshots/, never overwritten, so a
corrupt snapshot cannot destroy the last good one
* each archive is verified (opens as gzip, non-empty, config.yaml present)
BEFORE it is uploaded
* latest.txt ships in the SAME commit as the archive, so a reader can never see
a pointer to an object that failed to upload
* restore walks newest-first and falls back to an older snapshot when the
newest one does not yield a usable home directory
"""
import os
import shutil
import sqlite3
import sys
import tarfile
import tempfile
import time
from datetime import datetime, timezone
from pathlib import Path
from huggingface_hub import HfApi
BUCKET = os.environ.get("HERMES_BUCKET", "tkgray/hermes-data")
SNAP_DIR = "snapshots"
SNAP_PREFIX = f"{SNAP_DIR}/hermes-data-"
POINTER = "latest.txt"
LEGACY_OBJECT = "hermes-data.tgz" # pre-rotation single-object layout
RETAIN = int(os.environ.get("SNAPSHOT_RETAIN", "24"))
HOME = Path(os.environ.get("HERMES_HOME", "/opt/data"))
# Regenerable / runtime-only: never worth archiving, and .local is a whole
# uv-managed CPython (~5k files) that would dominate every snapshot.
SKIP_DIRS = {
".local", "cache", ".cache", "logs", "installs", "backups",
"node_modules", "__pycache__", "tmp", ".git", "gateway",
}
SKIP_SUFFIX = (".sock", ".lock", ".db-wal", ".db-shm", ".db-journal", ".pid")
def log(msg: str) -> None:
print(f"[bucket] {msg}", flush=True)
def _ignore(dirpath, names):
return [n for n in names if n in SKIP_DIRS or n.endswith(SKIP_SUFFIX)]
def _copy_db(src: Path, dst: Path) -> None:
"""Replace a possibly-torn file copy with a consistent sqlite backup."""
try:
with sqlite3.connect(f"file:{src}?mode=ro", uri=True, timeout=5) as s:
with sqlite3.connect(dst) as d:
s.backup(d)
except Exception as e: # locked / not-a-db: keep the plain copy
log(f"db backup degraded for {src.name}: {e}")
def _snapshot_names(api) -> list[str]:
try:
tree = api.list_bucket_tree(BUCKET, recursive=True)
except TypeError:
tree = api.list_bucket_tree(BUCKET)
names = []
for node in tree:
path = getattr(node, "path", None)
if path is None and isinstance(node, dict):
path = node.get("path")
if path is None:
path = str(node)
if path.startswith(SNAP_PREFIX) and path.endswith(".tgz"):
names.append(path)
return sorted(names)
def _verify_archive(path: Path) -> int:
"""Fail loudly rather than upload a snapshot we cannot read back."""
with tarfile.open(path, "r:gz") as tar:
members = tar.getmembers()
if not members:
raise ValueError("archive is empty")
if (HOME / "config.yaml").exists() and not any(
m.name.rstrip("/").endswith("config.yaml") for m in members
):
raise ValueError("archive is missing config.yaml")
return len(members)
def backup() -> int:
api = HfApi()
stage = Path(tempfile.mkdtemp(prefix="hermes-snap-"))
try:
target = stage / "data"
shutil.copytree(HOME, target, symlinks=True, ignore=_ignore, dirs_exist_ok=True)
for db in target.rglob("*.db"):
src = HOME / db.relative_to(target)
if src.is_file():
_copy_db(src, db)
archive = stage / "snap.tgz"
with tarfile.open(archive, "w:gz") as tar:
tar.add(target, arcname=".")
entries = _verify_archive(archive)
size = archive.stat().st_size
name = f"{SNAP_PREFIX}{datetime.now(timezone.utc):%Y%m%d-%H%M%S}.tgz"
# Archive + pointer in one commit: the pointer can never reference an
# object that did not make it.
api.batch_bucket_files(BUCKET, add=[(archive, name), (name.encode(), POINTER)])
log(f"backup ok: {size / 1e6:.1f} MB, {entries} entries, verified -> {name}")
names = _snapshot_names(api)
stale = names[:-RETAIN] if len(names) > RETAIN else []
if stale:
api.batch_bucket_files(BUCKET, delete=stale)
log(f"pruned {len(stale)} old snapshot(s); keeping newest {RETAIN}")
return 0
finally:
shutil.rmtree(stage, ignore_errors=True)
def _candidate_names(api) -> list[str]:
"""Pointer first, then every other snapshot newest-first, then the legacy object."""
ordered: list[str] = []
try:
with tempfile.TemporaryDirectory() as tmp:
local = Path(tmp) / "pointer"
api.download_bucket_files(BUCKET, [(POINTER, local)], raise_on_missing_files=True)
pointed = local.read_text(encoding="utf-8-sig").strip()
if pointed:
ordered.append(pointed)
except Exception as e:
log(f"no pointer ({type(e).__name__}); falling back to newest snapshot")
for name in reversed(_snapshot_names(api)):
if name not in ordered:
ordered.append(name)
# Pre-rotation layout: one object at the bucket root.
if LEGACY_OBJECT not in ordered:
ordered.append(LEGACY_OBJECT)
return ordered
def restore() -> int:
if (HOME / "config.yaml").exists():
log("restore skipped: /opt/data already populated")
return 0
api = HfApi()
for name in _candidate_names(api):
workdir = Path(tempfile.mkdtemp(prefix="hermes-restore-"))
local = workdir / "snap.tgz"
try:
api.download_bucket_files(BUCKET, [(name, local)], raise_on_missing_files=True)
with tarfile.open(local, "r:gz") as tar:
tar.extractall(HOME)
if (HOME / "config.yaml").exists():
log(f"restore ok: {name} -> {len(list(HOME.iterdir()))} entries in /opt/data")
return 0
log(f"restored {name} but no config.yaml; trying an older snapshot")
except Exception as e:
log(f"restore of {name} failed: {type(e).__name__}: {e}")
finally:
shutil.rmtree(workdir, ignore_errors=True)
log("no usable snapshot; starting fresh")
return 0
def main() -> int:
cmd = sys.argv[1] if len(sys.argv) > 1 else "backup"
if cmd == "restore":
return restore()
if cmd == "backup":
return backup()
if cmd == "loop":
interval = int(os.environ.get("SYNC_INTERVAL", "300"))
log(f"loop started: every {interval}s -> {BUCKET}/{SNAP_PREFIX}*.tgz (retain {RETAIN})")
while True:
time.sleep(interval)
try:
backup()
except Exception as e:
log(f"backup failed: {type(e).__name__}: {e}")
print(f"unknown command: {cmd}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())