Spaces:
Paused
Paused
fix: targeted restore — only sync user dirs, not system dirs
Browse filesPrevious approach (rsync entire rootfs → /) kept breaking /bin/sh
because HF dataset corrupts permissions and directory metadata.
New approach: only save and restore specific user directories:
/home, /root, /etc, /opt, /var, /usr/local
System directories (/bin, /sbin, /usr/bin, /usr/lib, etc.) come
from Docker image and are never touched. User-installed packages
are tracked via dpkg package list for reinstallation on restore.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
- ubuntu-server/git_sync_daemon.py +48 -43
ubuntu-server/git_sync_daemon.py
CHANGED
|
@@ -27,21 +27,24 @@ ROOTFS = os.path.join(PERSIST_PATH, "rootfs")
|
|
| 27 |
|
| 28 |
SYNC_INTERVAL = int(os.environ.get("SYNC_INTERVAL", "120"))
|
| 29 |
|
| 30 |
-
#
|
| 31 |
-
# System
|
| 32 |
-
# restoring them
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
"/
|
| 36 |
-
|
| 37 |
-
|
|
|
|
|
|
|
| 38 |
"/etc/hostname", "/etc/hosts", "/etc/resolv.conf", "/etc/mtab",
|
|
|
|
| 39 |
]
|
| 40 |
|
| 41 |
UPLOAD_IGNORE = [
|
| 42 |
"__pycache__", "*.pyc",
|
| 43 |
".git", ".git*",
|
| 44 |
-
"*.sock",
|
| 45 |
".huggingface",
|
| 46 |
".cache",
|
| 47 |
]
|
|
@@ -132,36 +135,34 @@ def download_dataset():
|
|
| 132 |
|
| 133 |
|
| 134 |
def restore_system():
|
| 135 |
-
"""Restore
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
log("── RESTORE: /data/rootfs/ → /")
|
| 139 |
if not os.path.isdir(ROOTFS):
|
| 140 |
log(f" ✗ {ROOTFS} does not exist, starting fresh")
|
| 141 |
os.makedirs(ROOTFS, exist_ok=True)
|
| 142 |
return
|
| 143 |
-
items = os.listdir(ROOTFS)
|
| 144 |
-
if not items:
|
| 145 |
-
log(f" ✗ {ROOTFS} is empty, starting fresh")
|
| 146 |
-
return
|
| 147 |
-
log(f" rootfs has {len(items)} top-level items: {items[:15]}")
|
| 148 |
-
log(f" rootfs size: {disk_usage(ROOTFS)}")
|
| 149 |
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
else:
|
| 164 |
-
log(f"
|
| 165 |
|
| 166 |
|
| 167 |
def restore_packages():
|
|
@@ -217,20 +218,24 @@ def save_and_upload():
|
|
| 217 |
from huggingface_hub import HfApi
|
| 218 |
|
| 219 |
log("══════════════════════════════════════════════")
|
| 220 |
-
log("── SAVE:
|
| 221 |
os.makedirs(ROOTFS, exist_ok=True)
|
| 222 |
save_package_list()
|
| 223 |
|
| 224 |
-
excludes = " ".join(f"--exclude='{e}'" for e in
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 229 |
log(f" rootfs size: {disk_usage(ROOTFS)}")
|
| 230 |
|
| 231 |
-
log(" chmod -R a+r rootfs/ ...")
|
| 232 |
-
run(f"chmod -R a+r {ROOTFS}/")
|
| 233 |
-
|
| 234 |
log("── UPLOAD: /data → dataset")
|
| 235 |
api = HfApi(token=HF_TOKEN)
|
| 236 |
ts = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
|
|
@@ -280,7 +285,7 @@ def cmd_init():
|
|
| 280 |
log(f" PERSIST_PATH: {PERSIST_PATH}")
|
| 281 |
log(f" ROOTFS: {ROOTFS}")
|
| 282 |
log(f" SYNC_INTERVAL: {SYNC_INTERVAL}s")
|
| 283 |
-
log(f"
|
| 284 |
|
| 285 |
if not HF_TOKEN or not HF_DATASET_REPO:
|
| 286 |
log(" ✗ No HF_TOKEN or HF_DATASET_REPO — persistence DISABLED")
|
|
|
|
| 27 |
|
| 28 |
SYNC_INTERVAL = int(os.environ.get("SYNC_INTERVAL", "120"))
|
| 29 |
|
| 30 |
+
# SAVE: only persist these directories (user data + config).
|
| 31 |
+
# System dirs (/bin, /lib, /usr/bin, etc.) come from Docker image;
|
| 32 |
+
# restoring them breaks permissions since HF git doesn't preserve mode bits.
|
| 33 |
+
PERSIST_DIRS = [
|
| 34 |
+
"/home", "/root", "/etc", "/opt", "/var",
|
| 35 |
+
"/usr/local",
|
| 36 |
+
]
|
| 37 |
+
|
| 38 |
+
# Paths to never touch (Docker-managed or virtual)
|
| 39 |
+
SAVE_EXCLUDES = [
|
| 40 |
"/etc/hostname", "/etc/hosts", "/etc/resolv.conf", "/etc/mtab",
|
| 41 |
+
"*.sock", "__pycache__", "*.pyc",
|
| 42 |
]
|
| 43 |
|
| 44 |
UPLOAD_IGNORE = [
|
| 45 |
"__pycache__", "*.pyc",
|
| 46 |
".git", ".git*",
|
| 47 |
+
"*.sock", "*.lock",
|
| 48 |
".huggingface",
|
| 49 |
".cache",
|
| 50 |
]
|
|
|
|
| 135 |
|
| 136 |
|
| 137 |
def restore_system():
|
| 138 |
+
"""Restore only user directories from /data/rootfs/ → /."""
|
| 139 |
+
log("── RESTORE: targeted directories only")
|
|
|
|
|
|
|
| 140 |
if not os.path.isdir(ROOTFS):
|
| 141 |
log(f" ✗ {ROOTFS} does not exist, starting fresh")
|
| 142 |
os.makedirs(ROOTFS, exist_ok=True)
|
| 143 |
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
|
| 145 |
+
excludes = " ".join(f"--exclude='{e}'" for e in SAVE_EXCLUDES)
|
| 146 |
+
restored = 0
|
| 147 |
+
for d in PERSIST_DIRS:
|
| 148 |
+
src = os.path.join(ROOTFS, d.lstrip("/"))
|
| 149 |
+
if not os.path.isdir(src):
|
| 150 |
+
continue
|
| 151 |
+
os.makedirs(d, exist_ok=True)
|
| 152 |
+
cmd = (f"rsync -rlD --no-specials --no-devices "
|
| 153 |
+
f"--no-perms --chmod=D755,F644 "
|
| 154 |
+
f"{excludes} {src}/ {d}/")
|
| 155 |
+
t0 = time.time()
|
| 156 |
+
rc, out = run(cmd)
|
| 157 |
+
elapsed = time.time() - t0
|
| 158 |
+
status = "OK" if rc == 0 else f"rc={rc}"
|
| 159 |
+
log(f" {d}: {status} ({elapsed:.1f}s)")
|
| 160 |
+
restored += 1
|
| 161 |
+
|
| 162 |
+
if restored == 0:
|
| 163 |
+
log(" No directories to restore (fresh start)")
|
| 164 |
else:
|
| 165 |
+
log(f" Restored {restored} directories")
|
| 166 |
|
| 167 |
|
| 168 |
def restore_packages():
|
|
|
|
| 218 |
from huggingface_hub import HfApi
|
| 219 |
|
| 220 |
log("══════════════════════════════════════════════")
|
| 221 |
+
log("── SAVE: targeted directories → /data/rootfs/")
|
| 222 |
os.makedirs(ROOTFS, exist_ok=True)
|
| 223 |
save_package_list()
|
| 224 |
|
| 225 |
+
excludes = " ".join(f"--exclude='{e}'" for e in SAVE_EXCLUDES)
|
| 226 |
+
t0_all = time.time()
|
| 227 |
+
for d in PERSIST_DIRS:
|
| 228 |
+
if not os.path.isdir(d):
|
| 229 |
+
continue
|
| 230 |
+
dst = os.path.join(ROOTFS, d.lstrip("/"))
|
| 231 |
+
os.makedirs(dst, exist_ok=True)
|
| 232 |
+
cmd = (f"rsync -rlD --no-specials --no-devices --delete "
|
| 233 |
+
f"{excludes} {d}/ {dst}/")
|
| 234 |
+
rc, out = run(cmd)
|
| 235 |
+
elapsed = time.time() - t0_all
|
| 236 |
+
log(f" rsync done ({elapsed:.1f}s)")
|
| 237 |
log(f" rootfs size: {disk_usage(ROOTFS)}")
|
| 238 |
|
|
|
|
|
|
|
|
|
|
| 239 |
log("── UPLOAD: /data → dataset")
|
| 240 |
api = HfApi(token=HF_TOKEN)
|
| 241 |
ts = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
|
|
|
|
| 285 |
log(f" PERSIST_PATH: {PERSIST_PATH}")
|
| 286 |
log(f" ROOTFS: {ROOTFS}")
|
| 287 |
log(f" SYNC_INTERVAL: {SYNC_INTERVAL}s")
|
| 288 |
+
log(f" PERSIST_DIRS: {PERSIST_DIRS}")
|
| 289 |
|
| 290 |
if not HF_TOKEN or not HF_DATASET_REPO:
|
| 291 |
log(" ✗ No HF_TOKEN or HF_DATASET_REPO — persistence DISABLED")
|