#!/usr/bin/env python3 """ Simple tar-gz backup of /opt/data → HF Dataset repo. Keeps the last 5 backups to prevent data loss from corruption. Env vars: HF_TOKEN HF access token with write permission HERMES_DATASET_REPO Target dataset repo, e.g. username/HermesFace-data HERMES_HOME Source directory (default: /opt/data) """ import os import shutil import sys import tarfile import tempfile from datetime import datetime from huggingface_hub import HfApi def main() -> None: repo_id = os.environ.get("HERMES_DATASET_REPO") token = os.environ.get("HF_TOKEN") state_dir = os.environ.get("HERMES_HOME", "/opt/data") if not repo_id or not token: print("[save_to_dataset] Missing HF_TOKEN or HERMES_DATASET_REPO.", file=sys.stderr) return if not os.path.isdir(state_dir): print(f"[save_to_dataset] No state directory to save: {state_dir}", file=sys.stderr) return api = HfApi(token=token) # Sync container logs into state dir for persistence try: sys_log_path = "/opt/data/logs" backup_log_path = os.path.join(state_dir, "logs/sys_logs") if os.path.exists(sys_log_path) and sys_log_path != backup_log_path: if os.path.exists(backup_log_path): shutil.rmtree(backup_log_path) shutil.copytree(sys_log_path, backup_log_path, ignore_dangling_symlinks=True) except Exception as e: print(f"[save_to_dataset] Warning: Failed to sync logs: {e}") timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") filename = f"state/backup-{timestamp}.tar.gz" with tempfile.TemporaryDirectory() as tmpdir: tar_path = os.path.join(tmpdir, "hermes.tar.gz") try: with tarfile.open(tar_path, "w:gz") as tf: def exclude(info: tarfile.TarInfo): bad = (".lock", ".tmp", ".pid", ".socket") if info.name.endswith(bad): return None if "__pycache__" in info.name: return None return info tf.add(state_dir, arcname=".", filter=exclude) except Exception as e: print(f"[save_to_dataset] Failed to compress: {e}", file=sys.stderr) return print(f"[save_to_dataset] Uploading backup: {filename}") try: api.upload_file( path_or_fileobj=tar_path, path_in_repo=filename, repo_id=repo_id, repo_type="dataset", ) except Exception as e: print(f"[save_to_dataset] Upload failed: {e}", file=sys.stderr) return try: files = api.list_repo_files(repo_id=repo_id, repo_type="dataset") backups = sorted( f for f in files if f.startswith("state/backup-") and (f.endswith(".tar") or f.endswith(".tar.gz")) ) if len(backups) > 5: to_delete = backups[:-5] print(f"[save_to_dataset] Rotating backups, deleting: {to_delete}") for old in to_delete: api.delete_file(path_in_repo=old, repo_id=repo_id, repo_type="dataset", token=token) except Exception as e: print(f"[save_to_dataset] Rotation failed (non-fatal): {e}", file=sys.stderr) if __name__ == "__main__": main()