feat: git_sync_daemon — gitfs-like auto-sync without FUSE

FUSE/gitfs not available in HF Spaces (no CAP_MKNOD).
Implements same behavior via pure git:
- git clone dataset → /data on startup
- rsync / → /data/rootfs/ periodically
- git add/commit/push automatically (like gitfs current/)
- Dataset mirrors filesystem at actual paths
- Configurable sync interval (default 120s)

Simplified start-server.sh: only starts services (sshd, ws-bridge,
ttyd, nginx). All persistence handled by git_sync_daemon.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Dockerfile

@@ -5,10 +5,11 @@ FROM ubuntu:24.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-# System + Python (for HuggingRun persistence sync)
+# System + Python + git-lfs (for HuggingRun git-based persistence)
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    ca-certificates curl wget python3 python3-pip python3-venv git \
+    ca-certificates curl wget python3 python3-pip python3-venv git git-lfs \
     && pip3 install --no-cache-dir --break-system-packages huggingface_hub websockets \
+    && git lfs install \
     && rm -rf /var/lib/apt/lists/*
 
 # Server: SSH + nginx + ttyd + tools
@@ -45,7 +46,7 @@ RUN mkdir -p /data
 # nginx + bridge + startup scripts
 COPY ubuntu-server/nginx.conf /etc/nginx/nginx.conf
 COPY ubuntu-server/ws-ssh-bridge.py /opt/ws-ssh-bridge.py
-COPY ubuntu-server/upload_sync.py /opt/upload_sync.py
+COPY ubuntu-server/git_sync_daemon.py /opt/git_sync_daemon.py
 COPY ubuntu-server/start-server.sh /opt/start-server.sh
 COPY scripts /scripts
 RUN chmod +x /scripts/entrypoint.sh /opt/start-server.sh

scripts/entrypoint.sh

@@ -1,95 +1,55 @@
 #!/bin/bash
 # ─────────────────────────────────────────────────────────────────────
 # HuggingRun Entrypoint
-# 1. Determine dataset repo, write env to /etc/huggingrun.env
-# 2. Download dataset → /data (snapshot_download)
-# 3. Run user command (start-server.sh sources the env and handles upload)
+# Uses git_sync_daemon.py (gitfs-like) for dataset ↔ filesystem sync
+# Then runs the user command (start-server.sh)
 # ─────────────────────────────────────────────────────────────────────
 set -e
-echo "[HuggingRun] Entrypoint — persistence + RUN_CMD"
-
-PERSIST_PATH="${PERSIST_PATH:-/data}"
-ENV_FILE="/etc/huggingrun.env"
-
-# Step 1: Download dataset and write env file
-python3 -u << 'PYEOF'
-import os, sys, traceback
-from pathlib import Path
-
-PERSIST_PATH = Path(os.environ.get("PERSIST_PATH", "/data"))
-HF_TOKEN = os.environ.get("HF_TOKEN", "")
-SPACE_ID = os.environ.get("SPACE_ID", "")
-HF_DATASET_REPO = os.environ.get("HF_DATASET_REPO", "")
-ENV_FILE = "/etc/huggingrun.env"
+echo "[HuggingRun] Entrypoint — git-sync persistence + RUN_CMD"
 
 # Determine dataset repo
-if not HF_DATASET_REPO:
-    if SPACE_ID:
-        HF_DATASET_REPO = f"{SPACE_ID}-data"
-    elif HF_TOKEN:
-        from huggingface_hub import HfApi
-        try:
-            HF_DATASET_REPO = HfApi(token=HF_TOKEN).whoami()["name"] + "/HuggingRun-data"
-        except:
-            pass
-
-# Write env file for other processes
-with open(ENV_FILE, "w") as f:
-    f.write(f'export HF_TOKEN="{HF_TOKEN}"\n')
-    f.write(f'export HF_DATASET_REPO="{HF_DATASET_REPO}"\n')
-    f.write(f'export PERSIST_PATH="{PERSIST_PATH}"\n')
-print(f"[HuggingRun] Wrote env to {ENV_FILE}")
-
-if not HF_TOKEN or not HF_DATASET_REPO:
-    print("[HuggingRun] No HF_TOKEN or dataset. Persistence disabled.")
-    PERSIST_PATH.mkdir(parents=True, exist_ok=True)
-    sys.exit(0)
-
-print(f"[HuggingRun] Dataset repo: {HF_DATASET_REPO}")
-
-from huggingface_hub import HfApi, snapshot_download
-
-api = HfApi(token=HF_TOKEN)
-
-# Ensure dataset exists
+if [ -z "$HF_DATASET_REPO" ]; then
+    if [ -n "$SPACE_ID" ]; then
+        export HF_DATASET_REPO="${SPACE_ID}-data"
+    elif [ -n "$HF_TOKEN" ]; then
+        export HF_DATASET_REPO=$(python3 -c "
+from huggingface_hub import HfApi
+import os
 try:
-    api.repo_info(repo_id=HF_DATASET_REPO, repo_type="dataset")
-    print(f"[HuggingRun] Dataset found: {HF_DATASET_REPO}")
+    print(HfApi(token=os.environ['HF_TOKEN']).whoami()['name'] + '/HuggingRun-data')
 except:
-    try:
-        api.create_repo(repo_id=HF_DATASET_REPO, repo_type="dataset", private=True)
-        print(f"[HuggingRun] Created dataset: {HF_DATASET_REPO}")
-    except Exception as e:
-        print(f"[HuggingRun] Cannot create dataset: {e}")
-        PERSIST_PATH.mkdir(parents=True, exist_ok=True)
-        sys.exit(0)
-
-# Check if dataset has files
-files = api.list_repo_files(repo_id=HF_DATASET_REPO, repo_type="dataset")
-data_files = [f for f in files if not f.startswith(".") and f != "README.md"]
+    print('')
+" 2>/dev/null)
+    fi
+fi
 
-if not data_files:
-    print("[HuggingRun] Dataset empty. Starting fresh.")
-    PERSIST_PATH.mkdir(parents=True, exist_ok=True)
-    sys.exit(0)
+# Ensure dataset repo exists
+if [ -n "$HF_TOKEN" ] && [ -n "$HF_DATASET_REPO" ]; then
+    python3 -c "
+from huggingface_hub import HfApi
+import os
+api = HfApi(token=os.environ['HF_TOKEN'])
+repo = os.environ['HF_DATASET_REPO']
+try:
+    api.repo_info(repo_id=repo, repo_type='dataset')
+    print(f'[HuggingRun] Dataset: {repo}')
+except:
+    api.create_repo(repo_id=repo, repo_type='dataset', private=True)
+    print(f'[HuggingRun] Created: {repo}')
+" 2>/dev/null || echo "[HuggingRun] Could not verify dataset"
+fi
 
-print(f"[HuggingRun] Downloading {len(data_files)} files from {HF_DATASET_REPO} ...")
-PERSIST_PATH.mkdir(parents=True, exist_ok=True)
-snapshot_download(
-    repo_id=HF_DATASET_REPO,
-    repo_type="dataset",
-    local_dir=str(PERSIST_PATH),
-    token=HF_TOKEN,
-    ignore_patterns=[".git*", "README.md"],
-)
-print("[HuggingRun] Download completed.")
-PYEOF
+# Run git-sync daemon (clones repo, restores system, starts background sync)
+python3 -u /opt/git_sync_daemon.py
 
-# Step 2: Source env and run user command
-if [ -f "$ENV_FILE" ]; then
-    source "$ENV_FILE"
-fi
+# Write env for other processes (sshd, cron, etc.)
+cat > /etc/huggingrun.env << ENVEOF
+export HF_TOKEN="${HF_TOKEN}"
+export HF_DATASET_REPO="${HF_DATASET_REPO}"
+export PERSIST_PATH="${PERSIST_PATH:-/data}"
+ENVEOF
 
+# Run user command
 CMD="${RUN_CMD:-python3 /app/demo_app.py}"
 echo "[HuggingRun] Running: $CMD"
 exec $CMD

ubuntu-server/git_sync_daemon.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+"""
+git_sync_daemon.py — gitfs-like auto-sync without FUSE
+
+Behavior (mirrors gitfs):
+  1. Clones HF dataset repo to /data
+  2. Periodically rsyncs filesystem → /data/rootfs/
+  3. Auto-commits and pushes changes via git
+  4. On restart: git pull → rsync /data/rootfs/ → /
+
+The dataset contains the actual filesystem at rootfs/usr/local/bin/..., etc.
+Every change is a git commit. Dataset = disk, always in sync.
+"""
+
+import os
+import sys
+import time
+import subprocess
+import signal
+import threading
+
+# ── Config ─────────────────────────────────────────────────────────
+PERSIST_PATH = os.environ.get("PERSIST_PATH", "/data")
+HF_TOKEN = os.environ.get("HF_TOKEN", "")
+HF_DATASET_REPO = os.environ.get("HF_DATASET_REPO", "")
+ROOTFS = os.path.join(PERSIST_PATH, "rootfs")
+
+SYNC_INTERVAL = int(os.environ.get("SYNC_INTERVAL", "120"))  # seconds between syncs
+COMMIT_NAME = "HuggingRun"
+COMMIT_EMAIL = "huggingrun@hf.space"
+
+RSYNC_EXCLUDES = [
+    "/proc", "/sys", "/dev", "/data", "/tmp", "/run",
+    "/mnt", "/media", "/snap",
+    "/var/cache/apt", "/var/lib/apt/lists",
+    "__pycache__", "*.pyc", "*.lock", "*.pid", "*.sock",
+]
+
+stop_event = threading.Event()
+
+
+def log(msg):
+    print(f"[git-sync] {msg}", file=sys.stderr, flush=True)
+
+
+def run(cmd, cwd=None, check=False):
+    """Run a shell command, return (returncode, stdout)."""
+    r = subprocess.run(
+        cmd, shell=True, cwd=cwd,
+        stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+        text=True,
+    )
+    if check and r.returncode != 0:
+        log(f"Command failed: {cmd}\n{r.stdout}")
+    return r.returncode, r.stdout.strip()
+
+
+def git_clone_or_pull():
+    """Clone the dataset repo to PERSIST_PATH, or pull if already cloned."""
+    repo_url = f"https://user:{HF_TOKEN}@huggingface.co/datasets/{HF_DATASET_REPO}"
+
+    if os.path.isdir(os.path.join(PERSIST_PATH, ".git")):
+        log("Git repo exists, pulling ...")
+        run(f"git -C {PERSIST_PATH} fetch origin main", cwd=PERSIST_PATH)
+        run(f"git -C {PERSIST_PATH} reset --hard origin/main", cwd=PERSIST_PATH)
+        log("Pull completed")
+    else:
+        log(f"Cloning {HF_DATASET_REPO} to {PERSIST_PATH} ...")
+        # Remove any existing non-git content
+        if os.path.exists(PERSIST_PATH):
+            run(f"rm -rf {PERSIST_PATH}")
+        rc, out = run(f"git clone --depth 1 {repo_url} {PERSIST_PATH}")
+        if rc != 0:
+            log(f"Clone failed: {out}")
+            os.makedirs(PERSIST_PATH, exist_ok=True)
+            # Init empty repo
+            run(f"git init {PERSIST_PATH}")
+            run(f"git -C {PERSIST_PATH} remote add origin {repo_url}")
+            run(f"git -C {PERSIST_PATH} checkout -b main")
+        log("Clone completed")
+
+    # Configure git
+    run(f'git config user.name "{COMMIT_NAME}"', cwd=PERSIST_PATH)
+    run(f'git config user.email "{COMMIT_EMAIL}"', cwd=PERSIST_PATH)
+    # Enable git-lfs
+    run("git lfs install --skip-smudge", cwd=PERSIST_PATH)
+
+
+def restore_system():
+    """Restore filesystem from /data/rootfs/ → /."""
+    if not os.path.isdir(ROOTFS) or not os.listdir(ROOTFS):
+        log("No rootfs found, starting fresh")
+        os.makedirs(ROOTFS, exist_ok=True)
+        return
+
+    log("Restoring filesystem from rootfs ...")
+    excludes = " ".join(f"--exclude='{e}'" for e in RSYNC_EXCLUDES)
+    run(f"rsync -aAX {excludes} --exclude='/scripts' {ROOTFS}/ /")
+    log("Filesystem restored")
+
+
+def save_and_push():
+    """Rsync filesystem → rootfs, git commit + push."""
+    log("Saving filesystem ...")
+    os.makedirs(ROOTFS, exist_ok=True)
+
+    # Rsync / → /data/rootfs/
+    excludes = " ".join(f"--exclude='{e}'" for e in RSYNC_EXCLUDES)
+    run(f"rsync -aAX --delete {excludes} / {ROOTFS}/")
+
+    # Fix permissions for git
+    run(f"chmod -R a+r {ROOTFS}/")
+
+    # Git add + commit + push
+    run("git add -A", cwd=PERSIST_PATH)
+
+    # Check if there are changes
+    rc, _ = run("git diff --cached --quiet", cwd=PERSIST_PATH)
+    if rc == 0:
+        log("No changes to commit")
+        return
+
+    ts = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
+    run(f'git commit -m "sync {ts}"', cwd=PERSIST_PATH)
+
+    log("Pushing to remote ...")
+    rc, out = run("git push origin main", cwd=PERSIST_PATH)
+    if rc != 0:
+        log(f"Push failed, trying force push: {out}")
+        rc, out = run("git push --force origin main", cwd=PERSIST_PATH)
+        if rc != 0:
+            log(f"Force push also failed: {out}")
+        else:
+            log("Force push succeeded")
+    else:
+        log("Push completed")
+
+
+def ensure_passwords():
+    """Re-ensure user/root passwords after restore."""
+    run("id user >/dev/null 2>&1 || useradd -m -s /bin/bash user")
+    run('usermod -p "$(openssl passwd -6 huggingrun)" user')
+    run('usermod -p "$(openssl passwd -6 huggingrun)" root')
+    run("ldconfig")
+    log("Passwords and ldconfig refreshed")
+
+
+def sync_loop():
+    """Background loop: periodically save and push."""
+    log(f"Sync loop started (interval={SYNC_INTERVAL}s)")
+    # Initial delay to let services start
+    if stop_event.wait(timeout=60):
+        return
+
+    while not stop_event.is_set():
+        try:
+            save_and_push()
+        except Exception as e:
+            log(f"Sync error: {e}")
+
+        if stop_event.wait(timeout=SYNC_INTERVAL):
+            break
+
+    # Final save on exit
+    log("Final save before exit ...")
+    try:
+        save_and_push()
+    except Exception as e:
+        log(f"Final save error: {e}")
+
+
+def main():
+    if not HF_TOKEN or not HF_DATASET_REPO:
+        log("No HF_TOKEN or HF_DATASET_REPO. Sync disabled.")
+        os.makedirs(PERSIST_PATH, exist_ok=True)
+        return
+
+    log(f"Starting git-sync daemon")
+    log(f"  Dataset: {HF_DATASET_REPO}")
+    log(f"  Persist: {PERSIST_PATH}")
+    log(f"  Interval: {SYNC_INTERVAL}s")
+
+    # Clone or pull
+    git_clone_or_pull()
+
+    # Restore system
+    restore_system()
+
+    # Re-set passwords
+    ensure_passwords()
+
+    # Start sync loop in background thread
+    sync_thread = threading.Thread(target=sync_loop, daemon=True)
+    sync_thread.start()
+
+    # Handle signals
+    def on_signal(sig, frame):
+        log(f"Signal {sig} received, stopping ...")
+        stop_event.set()
+        sync_thread.join(timeout=30)
+        sys.exit(0)
+
+    signal.signal(signal.SIGTERM, on_signal)
+    signal.signal(signal.SIGINT, on_signal)
+
+    # Keep main thread alive (will be used by start-server.sh exec)
+    log("Daemon ready. System restored.")
+
+
+if __name__ == "__main__":
+    main()

ubuntu-server/start-server.sh

@@ -1,94 +1,17 @@
 #!/bin/bash
 # ─────────────────────────────────────────────────────────────────────
-# HuggingRun Ubuntu Server: full system persistence
-# Port 7860 (nginx): web terminal + SSH-over-WebSocket
+# HuggingRun Ubuntu Server: ttyd + SSH-over-WebSocket
+# Port 7860 (nginx): web terminal + SSH
 #
-# Persistence model:
-#   /data/rootfs/ mirrors entire / filesystem
-#   upload_folder pushes /data/rootfs/ to HF dataset periodically
-#   Dataset shows actual filesystem paths: rootfs/usr/local/bin/...
+# Persistence is handled by git_sync_daemon.py (started in entrypoint)
+# This script just starts the services.
 # ─────────────────────────────────────────────────────────────────────
-echo "[start-server] Starting ..." >&2
-set -e
+echo "[start-server] Starting services ..." >&2
 
-# Source HuggingRun env (HF_TOKEN, HF_DATASET_REPO, PERSIST_PATH)
-[ -f /etc/huggingrun.env ] && source /etc/huggingrun.env
-
-export PERSIST_PATH="${PERSIST_PATH:-/data}"
 export SSH_PORT="${SSH_PORT:-2222}"
 export TTYD_PORT="${TTYD_PORT:-7681}"
 
-ROOTFS="$PERSIST_PATH/rootfs"
-
-# Rsync exclude list
-RSYNC_EXCLUDE=(
-    --exclude='/proc' --exclude='/sys' --exclude='/dev'
-    --exclude='/data' --exclude='/tmp' --exclude='/run'
-    --exclude='/mnt' --exclude='/media' --exclude='/snap'
-    --exclude='/var/cache/apt' --exclude='/var/lib/apt/lists'
-    --exclude='__pycache__' --exclude='*.pyc'
-    --exclude='.git'
-)
-
-# ── Phase 1: Restore system from dataset ──────────────────────────
-echo "[start-server] Restoring persisted system state ..." >&2
-if [ -d "$ROOTFS" ] && [ "$(ls -A "$ROOTFS" 2>/dev/null)" ]; then
-    echo "[persist] Restoring filesystem from $ROOTFS ..." >&2
-    rsync -aAX \
-        --exclude='/scripts' \
-        "${RSYNC_EXCLUDE[@]}" \
-        "$ROOTFS/" / 2>/dev/null || true
-    echo "[persist] Filesystem restored" >&2
-else
-    echo "[persist] No rootfs found, starting fresh" >&2
-    mkdir -p "$ROOTFS"
-fi
-
-# Always re-ensure passwords and user
-id user &>/dev/null || useradd -m -s /bin/bash user 2>/dev/null || true
-usermod -p "$(openssl passwd -6 huggingrun)" user 2>/dev/null || true
-usermod -p "$(openssl passwd -6 huggingrun)" root 2>/dev/null || true
-ldconfig 2>/dev/null || true
-echo "[persist] System ready" >&2
-
-# ── Phase 2: Save system (rsync + upload_folder) ─────────────────
-save_system() {
-    echo "[persist] Saving filesystem ..." >&2
-    mkdir -p "$ROOTFS"
-
-    # Rsync entire filesystem to /data/rootfs (exclude lock files etc.)
-    rsync -aAX --delete \
-        "${RSYNC_EXCLUDE[@]}" \
-        --exclude='*.lock' --exclude='*.pid' --exclude='*.sock' \
-        / "$ROOTFS/" 2>/dev/null || true
-
-    # Fix permissions so upload can read all files
-    chmod -R a+r "$ROOTFS/" 2>/dev/null || true
-
-    # Source env for credentials
-    [ -f /etc/huggingrun.env ] && source /etc/huggingrun.env
-
-    # Upload via huggingface_hub
-    python3 -u /opt/upload_sync.py 2>&1 || true
-
-    echo "[persist] Save completed at $(date -u +%H:%M:%S)" >&2
-}
-
-# Background: save every 300 seconds
-(
-    sleep 180  # initial delay (3 min, let system settle)
-    while true; do
-        save_system
-        sleep 300
-    done
-) &
-PERSIST_PID=$!
-echo "[start-server] Persistence background PID=$PERSIST_PID" >&2
-
-# Save on exit
-trap 'echo "[start-server] Saving before exit..." >&2; save_system; exit 0' SIGTERM SIGINT
-
-# ── Phase 3: Start sshd ──────────────────────────────────────────
+# ── sshd ──────────────────────────────────────────────────────────
 mkdir -p /run/sshd
 echo "[start-server] Starting sshd on 127.0.0.1:$SSH_PORT ..." >&2
 /usr/sbin/sshd -o "Port=$SSH_PORT" \
@@ -102,20 +25,20 @@ SSHD_PID=$!
 sleep 1
 echo "[start-server] sshd PID=$SSHD_PID" >&2
 
-# ── Phase 4: Start WebSocket-to-SSH bridge ────────────────────────
+# ── WebSocket-to-SSH bridge ──────────────────────────────────────
 echo "[start-server] Starting WS-SSH bridge on 127.0.0.1:7862 ..." >&2
 python3 /opt/ws-ssh-bridge.py &
 BRIDGE_PID=$!
 sleep 1
 echo "[start-server] WS-SSH bridge PID=$BRIDGE_PID" >&2
 
-# ── Phase 5: Start ttyd (web terminal) ───────────────────────────
+# ── ttyd (web terminal) ─────────────────────────────────────────
 echo "[start-server] Starting ttyd on 127.0.0.1:$TTYD_PORT ..." >&2
 ttyd --port "$TTYD_PORT" --writable --base-path / bash --login &
 TTYD_PID=$!
 sleep 1
 echo "[start-server] ttyd PID=$TTYD_PID" >&2
 
-# ── Phase 6: Start nginx (foreground, port 7860) ─────────────────
+# ── nginx (foreground, port 7860) ────────────────────────────────
 echo "[start-server] Starting nginx on 0.0.0.0:7860 ..." >&2
 exec nginx -g 'daemon off;'