Spaces:
Paused
Paused
File size: 3,385 Bytes
0790cf1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 | #!/usr/bin/env python3
"""
Simple tar-gz backup of /opt/data → HF Dataset repo.
Keeps the last 5 backups to prevent data loss from corruption.
Env vars:
HF_TOKEN HF access token with write permission
HERMES_DATASET_REPO Target dataset repo, e.g. username/HermesFace-data
HERMES_HOME Source directory (default: /opt/data)
"""
import os
import shutil
import sys
import tarfile
import tempfile
from datetime import datetime
from huggingface_hub import HfApi
def main() -> None:
repo_id = os.environ.get("HERMES_DATASET_REPO")
token = os.environ.get("HF_TOKEN")
state_dir = os.environ.get("HERMES_HOME", "/opt/data")
if not repo_id or not token:
print("[save_to_dataset] Missing HF_TOKEN or HERMES_DATASET_REPO.", file=sys.stderr)
return
if not os.path.isdir(state_dir):
print(f"[save_to_dataset] No state directory to save: {state_dir}", file=sys.stderr)
return
api = HfApi(token=token)
# Sync container logs into state dir for persistence
try:
sys_log_path = "/opt/data/logs"
backup_log_path = os.path.join(state_dir, "logs/sys_logs")
if os.path.exists(sys_log_path) and sys_log_path != backup_log_path:
if os.path.exists(backup_log_path):
shutil.rmtree(backup_log_path)
shutil.copytree(sys_log_path, backup_log_path, ignore_dangling_symlinks=True)
except Exception as e:
print(f"[save_to_dataset] Warning: Failed to sync logs: {e}")
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"state/backup-{timestamp}.tar.gz"
with tempfile.TemporaryDirectory() as tmpdir:
tar_path = os.path.join(tmpdir, "hermes.tar.gz")
try:
with tarfile.open(tar_path, "w:gz") as tf:
def exclude(info: tarfile.TarInfo):
bad = (".lock", ".tmp", ".pid", ".socket")
if info.name.endswith(bad):
return None
if "__pycache__" in info.name:
return None
return info
tf.add(state_dir, arcname=".", filter=exclude)
except Exception as e:
print(f"[save_to_dataset] Failed to compress: {e}", file=sys.stderr)
return
print(f"[save_to_dataset] Uploading backup: {filename}")
try:
api.upload_file(
path_or_fileobj=tar_path,
path_in_repo=filename,
repo_id=repo_id,
repo_type="dataset",
)
except Exception as e:
print(f"[save_to_dataset] Upload failed: {e}", file=sys.stderr)
return
try:
files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
backups = sorted(
f for f in files
if f.startswith("state/backup-") and (f.endswith(".tar") or f.endswith(".tar.gz"))
)
if len(backups) > 5:
to_delete = backups[:-5]
print(f"[save_to_dataset] Rotating backups, deleting: {to_delete}")
for old in to_delete:
api.delete_file(path_in_repo=old, repo_id=repo_id, repo_type="dataset", token=token)
except Exception as e:
print(f"[save_to_dataset] Rotation failed (non-fatal): {e}", file=sys.stderr)
if __name__ == "__main__":
main()
|