tao-shen Claude Opus 4.6 commited on
Commit
8bddfd6
Β·
1 Parent(s): 9f98401

fix: skip system dirs during restore to prevent /bin/sh permission loss

Browse files

HF datasets don't preserve Unix execute permissions. Restoring /bin/sh
from the dataset removes +x, breaking all shell=True subprocess calls.

- Save: still syncs full disk to dataset (user's full mirror requirement)
- Restore: skips /bin, /sbin, /lib, /usr/bin, /usr/lib, /usr/share etc.
(system packages reinstalled via user-packages.list + apt install)
- Only user-modifiable dirs (/home, /root, /etc, /opt, /var, /usr/local)
are restored from dataset

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Files changed (1) hide show
  1. entrypoint.py +25 -17
entrypoint.py CHANGED
@@ -29,21 +29,29 @@ LOGFILE = "/var/log/huggingrun.log"
29
  PKG_FILE = os.path.join(PERSIST_PATH, "user-packages.list")
30
  BASE_PKG_FILE = "/etc/base-packages.list"
31
 
32
- # rsync excludes for both save (/ β†’ /data/) and restore (/data/ β†’ /)
33
- RSYNC_EXCLUDES = [
34
- # Virtual / kernel (don't exist as real files)
35
- "/proc", "/sys", "/dev",
36
- # Our persist path (avoid recursion)
37
- "/data",
38
- # Temporary / runtime
39
- "/tmp", "/run",
40
- # Docker-managed (overwritten each container start)
41
- "/etc/hostname", "/etc/hosts", "/etc/resolv.conf", "/etc/mtab",
42
- # Transient
43
- "*.sock", "*.pid",
44
  "/var/lock",
45
  ]
46
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  # upload_folder ignore patterns (HF API rejects some paths)
48
  UPLOAD_IGNORE = [
49
  "__pycache__", "*.pyc",
@@ -160,11 +168,11 @@ def restore_state():
160
  log(" no filesystem data in dataset (fresh start)")
161
  return
162
 
163
- log("── RESTORE: rsync /data/ β†’ /")
164
- excludes = " ".join(f"--exclude='{e}'" for e in RSYNC_EXCLUDES)
165
  t0 = time.time()
166
  # rsync with -rlptD: recursive, links, perms, times, devices
167
- # --exclude .huggingface and .git (download metadata, not filesystem data)
168
  cmd = (f"rsync -rlptD --delete "
169
  f"{excludes} "
170
  f"--exclude='.huggingface' --exclude='.git' --exclude='.gitattributes' "
@@ -225,9 +233,9 @@ def save_and_upload():
225
  except Exception:
226
  pass
227
 
228
- # rsync entire filesystem β†’ /data/
229
  t0 = time.time()
230
- excludes = " ".join(f"--exclude='{e}'" for e in RSYNC_EXCLUDES)
231
  cmd = (f"rsync -rlptD --delete "
232
  f"{excludes} "
233
  f"--exclude='.huggingface' --exclude='.git' --exclude='.gitattributes' "
 
29
  PKG_FILE = os.path.join(PERSIST_PATH, "user-packages.list")
30
  BASE_PKG_FILE = "/etc/base-packages.list"
31
 
32
+ # Common rsync excludes
33
+ _RSYNC_COMMON = [
34
+ "/proc", "/sys", "/dev", # virtual / kernel
35
+ "/data", # our persist path (avoid recursion)
36
+ "/tmp", "/run", # temporary / runtime
37
+ "/etc/hostname", "/etc/hosts", "/etc/resolv.conf", "/etc/mtab", # Docker-managed
38
+ "*.sock", "*.pid", # transient
 
 
 
 
 
39
  "/var/lock",
40
  ]
41
 
42
+ # SAVE excludes: minimal β€” upload full disk mirror to dataset
43
+ RSYNC_SAVE_EXCLUDES = list(_RSYNC_COMMON)
44
+
45
+ # RESTORE excludes: also skip system dirs (HF loses Unix execute permissions,
46
+ # restoring /bin/sh without +x breaks the container). System packages are
47
+ # reinstalled via user-packages.list instead.
48
+ RSYNC_RESTORE_EXCLUDES = list(_RSYNC_COMMON) + [
49
+ "/bin", "/sbin", "/lib", "/lib64",
50
+ "/usr/bin", "/usr/sbin", "/usr/lib", "/usr/libexec", "/usr/include",
51
+ "/usr/share", # system data, reinstalled via apt
52
+ "/boot", # kernel files
53
+ ]
54
+
55
  # upload_folder ignore patterns (HF API rejects some paths)
56
  UPLOAD_IGNORE = [
57
  "__pycache__", "*.pyc",
 
168
  log(" no filesystem data in dataset (fresh start)")
169
  return
170
 
171
+ log("── RESTORE: rsync /data/ β†’ / (skipping system dirs)")
172
+ excludes = " ".join(f"--exclude='{e}'" for e in RSYNC_RESTORE_EXCLUDES)
173
  t0 = time.time()
174
  # rsync with -rlptD: recursive, links, perms, times, devices
175
+ # Skip system binary dirs (HF loses execute permissions); packages reinstalled via apt
176
  cmd = (f"rsync -rlptD --delete "
177
  f"{excludes} "
178
  f"--exclude='.huggingface' --exclude='.git' --exclude='.gitattributes' "
 
233
  except Exception:
234
  pass
235
 
236
+ # rsync entire filesystem β†’ /data/ (full mirror)
237
  t0 = time.time()
238
+ excludes = " ".join(f"--exclude='{e}'" for e in RSYNC_SAVE_EXCLUDES)
239
  cmd = (f"rsync -rlptD --delete "
240
  f"{excludes} "
241
  f"--exclude='.huggingface' --exclude='.git' --exclude='.gitattributes' "