tao-shen Claude Opus 4.6 commited on
Commit
9161deb
·
1 Parent(s): c44ae59

fix: targeted restore — only sync user dirs, not system dirs

Browse files

Previous approach (rsync entire rootfs → /) kept breaking /bin/sh
because HF dataset corrupts permissions and directory metadata.

New approach: only save and restore specific user directories:
/home, /root, /etc, /opt, /var, /usr/local

System directories (/bin, /sbin, /usr/bin, /usr/lib, etc.) come
from Docker image and are never touched. User-installed packages
are tracked via dpkg package list for reinstallation on restore.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Files changed (1) hide show
  1. ubuntu-server/git_sync_daemon.py +48 -43
ubuntu-server/git_sync_daemon.py CHANGED
@@ -27,21 +27,24 @@ ROOTFS = os.path.join(PERSIST_PATH, "rootfs")
27
 
28
  SYNC_INTERVAL = int(os.environ.get("SYNC_INTERVAL", "120"))
29
 
30
- # Exclude: kernel vfs, Docker-managed, and base system directories.
31
- # System binaries (/bin, /sbin, /lib*) come from the Docker image build;
32
- # restoring them from HF Dataset breaks permissions (git doesn't preserve mode bits).
33
- # User-installed packages are tracked separately via dpkg package list.
34
- RSYNC_EXCLUDES = [
35
- "/proc", "/sys", "/dev", "/data", "/run",
36
- "/bin", "/sbin", "/lib", "/lib64",
37
- "/usr/bin", "/usr/sbin", "/usr/lib",
 
 
38
  "/etc/hostname", "/etc/hosts", "/etc/resolv.conf", "/etc/mtab",
 
39
  ]
40
 
41
  UPLOAD_IGNORE = [
42
  "__pycache__", "*.pyc",
43
  ".git", ".git*",
44
- "*.sock",
45
  ".huggingface",
46
  ".cache",
47
  ]
@@ -132,36 +135,34 @@ def download_dataset():
132
 
133
 
134
  def restore_system():
135
- """Restore filesystem from /data/rootfs/ → /."""
136
- import stat
137
-
138
- log("── RESTORE: /data/rootfs/ → /")
139
  if not os.path.isdir(ROOTFS):
140
  log(f" ✗ {ROOTFS} does not exist, starting fresh")
141
  os.makedirs(ROOTFS, exist_ok=True)
142
  return
143
- items = os.listdir(ROOTFS)
144
- if not items:
145
- log(f" ✗ {ROOTFS} is empty, starting fresh")
146
- return
147
- log(f" rootfs has {len(items)} top-level items: {items[:15]}")
148
- log(f" rootfs size: {disk_usage(ROOTFS)}")
149
 
150
- # rsync with --no-perms --chmod to force sane permissions.
151
- # HF Dataset (git) doesn't preserve Unix permissions, so downloaded
152
- # files/dirs have wrong modes. Without --chmod, rsync would copy
153
- # broken permissions (e.g. /usr/ with 0644) making /bin/sh inaccessible.
154
- excludes = " ".join(f"--exclude='{e}'" for e in RSYNC_EXCLUDES)
155
- cmd = (f"rsync -rlD --no-specials --no-devices "
156
- f"--no-perms --chmod=D755,F644 "
157
- f"{excludes} {ROOTFS}/ /")
158
- t0 = time.time()
159
- rc, out = run(cmd)
160
- elapsed = time.time() - t0
161
- if rc == 0:
162
- log(f" Restore completed ({elapsed:.1f}s)")
 
 
 
 
 
 
163
  else:
164
- log(f" Restore had issues (rc={rc}, {elapsed:.1f}s)")
165
 
166
 
167
  def restore_packages():
@@ -217,20 +218,24 @@ def save_and_upload():
217
  from huggingface_hub import HfApi
218
 
219
  log("══════════════════════════════════════════════")
220
- log("── SAVE: rsync / → /data/rootfs/")
221
  os.makedirs(ROOTFS, exist_ok=True)
222
  save_package_list()
223
 
224
- excludes = " ".join(f"--exclude='{e}'" for e in RSYNC_EXCLUDES)
225
- t0 = time.time()
226
- rc, out = run(f"rsync -rlptgoD --no-specials --no-devices --delete {excludes} / {ROOTFS}/")
227
- elapsed = time.time() - t0
228
- log(f" rsync done (rc={rc}, {elapsed:.1f}s)")
 
 
 
 
 
 
 
229
  log(f" rootfs size: {disk_usage(ROOTFS)}")
230
 
231
- log(" chmod -R a+r rootfs/ ...")
232
- run(f"chmod -R a+r {ROOTFS}/")
233
-
234
  log("── UPLOAD: /data → dataset")
235
  api = HfApi(token=HF_TOKEN)
236
  ts = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
@@ -280,7 +285,7 @@ def cmd_init():
280
  log(f" PERSIST_PATH: {PERSIST_PATH}")
281
  log(f" ROOTFS: {ROOTFS}")
282
  log(f" SYNC_INTERVAL: {SYNC_INTERVAL}s")
283
- log(f" RSYNC_EXCLUDES: {RSYNC_EXCLUDES}")
284
 
285
  if not HF_TOKEN or not HF_DATASET_REPO:
286
  log(" ✗ No HF_TOKEN or HF_DATASET_REPO — persistence DISABLED")
 
27
 
28
  SYNC_INTERVAL = int(os.environ.get("SYNC_INTERVAL", "120"))
29
 
30
+ # SAVE: only persist these directories (user data + config).
31
+ # System dirs (/bin, /lib, /usr/bin, etc.) come from Docker image;
32
+ # restoring them breaks permissions since HF git doesn't preserve mode bits.
33
+ PERSIST_DIRS = [
34
+ "/home", "/root", "/etc", "/opt", "/var",
35
+ "/usr/local",
36
+ ]
37
+
38
+ # Paths to never touch (Docker-managed or virtual)
39
+ SAVE_EXCLUDES = [
40
  "/etc/hostname", "/etc/hosts", "/etc/resolv.conf", "/etc/mtab",
41
+ "*.sock", "__pycache__", "*.pyc",
42
  ]
43
 
44
  UPLOAD_IGNORE = [
45
  "__pycache__", "*.pyc",
46
  ".git", ".git*",
47
+ "*.sock", "*.lock",
48
  ".huggingface",
49
  ".cache",
50
  ]
 
135
 
136
 
137
  def restore_system():
138
+ """Restore only user directories from /data/rootfs/ → /."""
139
+ log("── RESTORE: targeted directories only")
 
 
140
  if not os.path.isdir(ROOTFS):
141
  log(f" ✗ {ROOTFS} does not exist, starting fresh")
142
  os.makedirs(ROOTFS, exist_ok=True)
143
  return
 
 
 
 
 
 
144
 
145
+ excludes = " ".join(f"--exclude='{e}'" for e in SAVE_EXCLUDES)
146
+ restored = 0
147
+ for d in PERSIST_DIRS:
148
+ src = os.path.join(ROOTFS, d.lstrip("/"))
149
+ if not os.path.isdir(src):
150
+ continue
151
+ os.makedirs(d, exist_ok=True)
152
+ cmd = (f"rsync -rlD --no-specials --no-devices "
153
+ f"--no-perms --chmod=D755,F644 "
154
+ f"{excludes} {src}/ {d}/")
155
+ t0 = time.time()
156
+ rc, out = run(cmd)
157
+ elapsed = time.time() - t0
158
+ status = "OK" if rc == 0 else f"rc={rc}"
159
+ log(f" {d}: {status} ({elapsed:.1f}s)")
160
+ restored += 1
161
+
162
+ if restored == 0:
163
+ log(" No directories to restore (fresh start)")
164
  else:
165
+ log(f" Restored {restored} directories")
166
 
167
 
168
  def restore_packages():
 
218
  from huggingface_hub import HfApi
219
 
220
  log("══════════════════════════════════════════════")
221
+ log("── SAVE: targeted directories → /data/rootfs/")
222
  os.makedirs(ROOTFS, exist_ok=True)
223
  save_package_list()
224
 
225
+ excludes = " ".join(f"--exclude='{e}'" for e in SAVE_EXCLUDES)
226
+ t0_all = time.time()
227
+ for d in PERSIST_DIRS:
228
+ if not os.path.isdir(d):
229
+ continue
230
+ dst = os.path.join(ROOTFS, d.lstrip("/"))
231
+ os.makedirs(dst, exist_ok=True)
232
+ cmd = (f"rsync -rlD --no-specials --no-devices --delete "
233
+ f"{excludes} {d}/ {dst}/")
234
+ rc, out = run(cmd)
235
+ elapsed = time.time() - t0_all
236
+ log(f" rsync done ({elapsed:.1f}s)")
237
  log(f" rootfs size: {disk_usage(ROOTFS)}")
238
 
 
 
 
239
  log("── UPLOAD: /data → dataset")
240
  api = HfApi(token=HF_TOKEN)
241
  ts = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
 
285
  log(f" PERSIST_PATH: {PERSIST_PATH}")
286
  log(f" ROOTFS: {ROOTFS}")
287
  log(f" SYNC_INTERVAL: {SYNC_INTERVAL}s")
288
+ log(f" PERSIST_DIRS: {PERSIST_DIRS}")
289
 
290
  if not HF_TOKEN or not HF_DATASET_REPO:
291
  log(" ✗ No HF_TOKEN or HF_DATASET_REPO — persistence DISABLED")