feat: direct file sync — dataset root = filesystem root

No more tar.zst. Dataset mirrors the container's / directly:
- SAVE: rsync / → /data/ → upload_folder to dataset
- RESTORE: snapshot_download → rsync /data/ → /

Dataset browsable: bin/, etc/, home/, usr/, etc. match the OS.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

entrypoint.py

@@ -2,12 +2,10 @@
 """
 HuggingRun v2 — Single entrypoint for Ubuntu Server on HuggingFace Spaces.
 
-Replaces: entrypoint.sh, start-server.sh, git_sync_daemon.py, log_streamer.py
-
-Persistence: full filesystem sync via tar.zst ↔ HF Dataset.
-  Dataset = disk. Container state is fully mirrored to the dataset.
-  On startup:  download state.tar.zst → extract to /
-  On sync:     tar selected dirs → upload state.tar.zst
+Persistence: dataset root = filesystem root (direct file mirror).
+  Dataset directory structure is identical to the container's /.
+  On startup:  snapshot_download → rsync /data/ → /
+  On sync:     rsync / → /data/ → upload_folder
 """
 
 import http.server
@@ -28,32 +26,31 @@ SYNC_INTERVAL = int(os.environ.get("SYNC_INTERVAL", "60"))
 SSH_PORT = os.environ.get("SSH_PORT", "2222")
 TTYD_PORT = os.environ.get("TTYD_PORT", "7681")
 LOGFILE = "/var/log/huggingrun.log"
-STATE_FILE = os.path.join(PERSIST_PATH, "state.tar.zst")
 PKG_FILE = os.path.join(PERSIST_PATH, "user-packages.list")
 BASE_PKG_FILE = "/etc/base-packages.list"
 
-# Full-disk persistence: tar everything from / except virtual/transient dirs.
-# Excludes only things that CANNOT or SHOULD NOT be persisted.
-TAR_EXCLUDES = [
-    # Virtual filesystems (kernel-provided, not real files)
-    "proc",
-    "sys",
-    "dev",
-    # Our own persist path (avoid recursive tar of state.tar.zst)
-    "data",
-    # Temporary
-    "tmp",
-    "run",
+# rsync excludes for both save (/ → /data/) and restore (/data/ → /)
+RSYNC_EXCLUDES = [
+    # Virtual / kernel (don't exist as real files)
+    "/proc", "/sys", "/dev",
+    # Our persist path (avoid recursion)
+    "/data",
+    # Temporary / runtime
+    "/tmp", "/run",
     # Docker-managed (overwritten each container start)
-    "etc/hostname",
-    "etc/hosts",
-    "etc/resolv.conf",
-    "etc/mtab",
-    # Transient runtime
-    "var/lock",
-    # Sockets, pids
-    "*.sock",
-    "*.pid",
+    "/etc/hostname", "/etc/hosts", "/etc/resolv.conf", "/etc/mtab",
+    # Transient
+    "*.sock", "*.pid",
+    "/var/lock",
+]
+
+# upload_folder ignore patterns (HF API rejects some paths)
+UPLOAD_IGNORE = [
+    "__pycache__", "*.pyc",
+    ".git", ".git*",
+    "*.sock", "*.lock",
+    ".huggingface",
+    ".cache",
 ]
 
 
@@ -136,52 +133,49 @@ def restore_state():
         os.makedirs(PERSIST_PATH, exist_ok=True)
         return
 
-    log("── RESTORE: downloading state")
+    log("── RESTORE: downloading dataset → /data/")
     os.makedirs(PERSIST_PATH, exist_ok=True)
     t0 = time.time()
     try:
-        from huggingface_hub import hf_hub_download
-        path = hf_hub_download(
+        from huggingface_hub import snapshot_download
+        snapshot_download(
             repo_id=HF_DATASET_REPO,
-            filename="state.tar.zst",
             repo_type="dataset",
             local_dir=PERSIST_PATH,
             token=HF_TOKEN,
         )
         elapsed = time.time() - t0
-        size_mb = os.path.getsize(path) / 1024 / 1024
-        log(f"  downloaded {size_mb:.1f}MB ({elapsed:.1f}s)")
+        log(f"  downloaded ({elapsed:.1f}s)")
     except Exception as e:
-        log(f"  no state to restore: {e}")
+        log(f"  download failed or empty: {e}")
         return
 
-    log("── RESTORE: extracting state.tar.zst → /")
+    # Check if there's actual filesystem data (look for top-level dirs like bin/, etc/)
+    has_data = any(
+        os.path.isdir(os.path.join(PERSIST_PATH, d))
+        for d in ["bin", "etc", "home", "usr", "root"]
+    )
+    if not has_data:
+        log("  no filesystem data in dataset (fresh start)")
+        return
+
+    log("── RESTORE: rsync /data/ → /")
+    excludes = " ".join(f"--exclude='{e}'" for e in RSYNC_EXCLUDES)
     t0 = time.time()
-    rc, out = run(f"tar --zstd -xpf {STATE_FILE} -C /")
+    # rsync with -rlptD: recursive, links, perms, times, devices
+    # --exclude .huggingface and .git (download metadata, not filesystem data)
+    cmd = (f"rsync -rlptD --delete "
+           f"{excludes} "
+           f"--exclude='.huggingface' --exclude='.git' --exclude='.gitattributes' "
+           f"--exclude='user-packages.list' "
+           f"{PERSIST_PATH}/ /")
+    rc, out = run(cmd)
     elapsed = time.time() - t0
     if rc == 0:
-        # Show what was restored
-        rc2, listing = run(f"tar --zstd -tf {STATE_FILE} | head -1 | cut -d/ -f1 | sort -u")
-        log(f"  extracted ({elapsed:.1f}s)")
-        # Count files
-        rc3, count = run(f"tar --zstd -tf {STATE_FILE} | wc -l")
-        if rc3 == 0:
-            log(f"  {count.strip()} files restored")
+        rc2, count = run(f"find {PERSIST_PATH} -type f | wc -l")
+        log(f"  restored ({elapsed:.1f}s), {count.strip()} files")
     else:
-        log(f"  extraction failed ({elapsed:.1f}s)")
-
-    # Also download package list if it exists
-    try:
-        from huggingface_hub import hf_hub_download
-        hf_hub_download(
-            repo_id=HF_DATASET_REPO,
-            filename="user-packages.list",
-            repo_type="dataset",
-            local_dir=PERSIST_PATH,
-            token=HF_TOKEN,
-        )
-    except Exception:
-        pass
+        log(f"  restore failed ({elapsed:.1f}s)")
 
 
 def restore_packages():
@@ -215,6 +209,7 @@ def ensure_passwords():
 def save_and_upload():
     if not HF_TOKEN or not HF_DATASET_REPO:
         return
+    import shutil
     from huggingface_hub import HfApi
 
     log("══ SYNC: save + upload ══")
@@ -229,54 +224,54 @@ def save_and_upload():
     except Exception:
         pass
 
-    # Create tar.zst of entire filesystem (full-disk persistence)
+    # rsync entire filesystem → /data/
     t0 = time.time()
-    excludes = " ".join(f"--exclude='{e}'" for e in TAR_EXCLUDES)
-    cmd = f"tar --zstd {excludes} -cpf {STATE_FILE} -C / ."
+    excludes = " ".join(f"--exclude='{e}'" for e in RSYNC_EXCLUDES)
+    cmd = (f"rsync -rlptD --delete "
+           f"{excludes} "
+           f"--exclude='.huggingface' --exclude='.git' --exclude='.gitattributes' "
+           f"--exclude='user-packages.list' "
+           f"/ {PERSIST_PATH}/")
     rc, out = run(cmd)
     elapsed = time.time() - t0
     if rc != 0:
-        # tar returns 1 for "file changed as we read it" — that's OK
-        if rc == 1:
-            log(f"  tar: warnings (files changed during archive) ({elapsed:.1f}s)")
-        else:
-            log(f"  tar failed rc={rc} ({elapsed:.1f}s)")
-            return
-
-    size_mb = os.path.getsize(STATE_FILE) / 1024 / 1024
-    log(f"  tar: {size_mb:.1f}MB ({elapsed:.1f}s) [full disk]")
-
-    # Upload both files
+        log(f"  rsync failed ({elapsed:.1f}s)")
+        return
+    log(f"  rsync → /data/ ({elapsed:.1f}s)")
+
+    # Clean dirs that HF API rejects
+    for reject_dir in [".cache"]:
+        for dirpath, dirnames, filenames in os.walk(PERSIST_PATH):
+            for d in list(dirnames):
+                if d == reject_dir:
+                    full = os.path.join(dirpath, d)
+                    log(f"  rm {full}")
+                    shutil.rmtree(full, ignore_errors=True)
+                    dirnames.remove(d)
+
+    # Count files to upload
+    rc2, count = run(f"find {PERSIST_PATH} -type f -not -path '*/.git/*' -not -path '*/.huggingface/*' | wc -l")
+    log(f"  {count.strip()} files to upload")
+
+    # Upload folder
     api = HfApi(token=HF_TOKEN)
     ts = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
 
     t0 = time.time()
     try:
-        api.upload_file(
-            path_or_fileobj=STATE_FILE,
-            path_in_repo="state.tar.zst",
+        api.upload_folder(
+            folder_path=PERSIST_PATH,
             repo_id=HF_DATASET_REPO,
             repo_type="dataset",
+            path_in_repo="",
             commit_message=f"sync {ts}",
+            ignore_patterns=UPLOAD_IGNORE,
         )
         elapsed = time.time() - t0
-        log(f"  uploaded state.tar.zst ({elapsed:.1f}s)")
+        log(f"  uploaded ({elapsed:.1f}s)")
     except Exception as e:
-        log(f"  upload state.tar.zst failed: {e}")
-
-    # Upload package list (separate commit to avoid conflicts)
-    if os.path.exists(PKG_FILE):
-        try:
-            api.upload_file(
-                path_or_fileobj=PKG_FILE,
-                path_in_repo="user-packages.list",
-                repo_id=HF_DATASET_REPO,
-                repo_type="dataset",
-                commit_message=f"packages {ts}",
-            )
-            log("  uploaded user-packages.list")
-        except Exception as e:
-            log(f"  upload packages failed: {e}")
+        elapsed = time.time() - t0
+        log(f"  upload failed ({elapsed:.1f}s): {e}")
 
     log("══ SYNC: done ══")