Spaces:
Paused
Paused
fix: restore original logging pattern + fix sync hanging
Browse filesRoot cause analysis:
1. sync daemon log() changed from stderr to stdout (broke HF log capture)
2. stdbuf -oL -eL removed from RUN_CMD (broke line buffering)
3. start-server.sh used exec > >(tee >&2) instead of direct echo >&2
4. upload_large_folder hangs for hours on 1.8GB rootfs
Fixes:
- Restore log() to use file=sys.stderr (original design from ea21d45)
- Restore stdbuf -oL -eL in Dockerfile RUN_CMD
- Simplify start-server.sh to direct echo >&2 + logfile write
- Switch from upload_large_folder to upload_folder (doesn't hang)
- Restore exec nginx (original PID 1 = nginx, not bash+wait)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
- Dockerfile +1 -1
- ubuntu-server/git_sync_daemon.py +7 -20
- ubuntu-server/start-server.sh +5 -16
Dockerfile
CHANGED
|
@@ -55,7 +55,7 @@ RUN chmod +x /scripts/entrypoint.sh /scripts/start-server.sh
|
|
| 55 |
|
| 56 |
ENV PERSIST_PATH=/data
|
| 57 |
ENV PYTHONUNBUFFERED=1
|
| 58 |
-
ENV RUN_CMD="/scripts/start-server.sh"
|
| 59 |
ENV SSH_PORT=2222
|
| 60 |
|
| 61 |
# Run as root (needed for: apt install persistence, bind mounts, sshd)
|
|
|
|
| 55 |
|
| 56 |
ENV PERSIST_PATH=/data
|
| 57 |
ENV PYTHONUNBUFFERED=1
|
| 58 |
+
ENV RUN_CMD="stdbuf -oL -eL /scripts/start-server.sh"
|
| 59 |
ENV SSH_PORT=2222
|
| 60 |
|
| 61 |
# Run as root (needed for: apt install persistence, bind mounts, sshd)
|
ubuntu-server/git_sync_daemon.py
CHANGED
|
@@ -18,6 +18,7 @@ import time
|
|
| 18 |
import subprocess
|
| 19 |
import signal
|
| 20 |
import shutil
|
|
|
|
| 21 |
|
| 22 |
# ββ Config βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 23 |
PERSIST_PATH = os.environ.get("PERSIST_PATH", "/data")
|
|
@@ -57,7 +58,7 @@ LOGFILE = "/var/log/huggingrun.log"
|
|
| 57 |
def log(msg):
|
| 58 |
ts = time.strftime("%H:%M:%S", time.gmtime())
|
| 59 |
line = f"[sync {ts}] {msg}"
|
| 60 |
-
print(line, flush=True)
|
| 61 |
try:
|
| 62 |
with open(LOGFILE, "a") as f:
|
| 63 |
f.write(line + "\n")
|
|
@@ -244,38 +245,24 @@ def save_and_upload():
|
|
| 244 |
log("ββ UPLOAD: /data β dataset")
|
| 245 |
api = HfApi(token=HF_TOKEN)
|
| 246 |
ts = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
|
| 247 |
-
log(f" repo={HF_DATASET_REPO}, method=
|
| 248 |
log(f" ignore={UPLOAD_IGNORE}")
|
| 249 |
|
| 250 |
t0 = time.time()
|
| 251 |
try:
|
| 252 |
-
api.
|
| 253 |
folder_path=PERSIST_PATH,
|
| 254 |
repo_id=HF_DATASET_REPO,
|
| 255 |
repo_type="dataset",
|
|
|
|
|
|
|
| 256 |
ignore_patterns=UPLOAD_IGNORE,
|
| 257 |
)
|
| 258 |
elapsed = time.time() - t0
|
| 259 |
log(f" β Upload completed ({elapsed:.1f}s)")
|
| 260 |
except Exception as e:
|
| 261 |
elapsed = time.time() - t0
|
| 262 |
-
log(f" β
|
| 263 |
-
log(" Trying fallback: upload_folder ...")
|
| 264 |
-
t0 = time.time()
|
| 265 |
-
try:
|
| 266 |
-
api.upload_folder(
|
| 267 |
-
folder_path=PERSIST_PATH,
|
| 268 |
-
repo_id=HF_DATASET_REPO,
|
| 269 |
-
repo_type="dataset",
|
| 270 |
-
path_in_repo="",
|
| 271 |
-
commit_message=f"sync {ts}",
|
| 272 |
-
ignore_patterns=UPLOAD_IGNORE,
|
| 273 |
-
)
|
| 274 |
-
elapsed = time.time() - t0
|
| 275 |
-
log(f" β Upload completed via fallback ({elapsed:.1f}s)")
|
| 276 |
-
except Exception as e2:
|
| 277 |
-
elapsed = time.time() - t0
|
| 278 |
-
log(f" β Fallback also failed ({elapsed:.1f}s): {e2}")
|
| 279 |
|
| 280 |
log("ββββββββββββββββββββββββββββββββββββββββββββββ")
|
| 281 |
|
|
|
|
| 18 |
import subprocess
|
| 19 |
import signal
|
| 20 |
import shutil
|
| 21 |
+
import threading
|
| 22 |
|
| 23 |
# ββ Config βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 24 |
PERSIST_PATH = os.environ.get("PERSIST_PATH", "/data")
|
|
|
|
| 58 |
def log(msg):
|
| 59 |
ts = time.strftime("%H:%M:%S", time.gmtime())
|
| 60 |
line = f"[sync {ts}] {msg}"
|
| 61 |
+
print(line, file=sys.stderr, flush=True)
|
| 62 |
try:
|
| 63 |
with open(LOGFILE, "a") as f:
|
| 64 |
f.write(line + "\n")
|
|
|
|
| 245 |
log("ββ UPLOAD: /data β dataset")
|
| 246 |
api = HfApi(token=HF_TOKEN)
|
| 247 |
ts = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
|
| 248 |
+
log(f" repo={HF_DATASET_REPO}, method=upload_folder")
|
| 249 |
log(f" ignore={UPLOAD_IGNORE}")
|
| 250 |
|
| 251 |
t0 = time.time()
|
| 252 |
try:
|
| 253 |
+
api.upload_folder(
|
| 254 |
folder_path=PERSIST_PATH,
|
| 255 |
repo_id=HF_DATASET_REPO,
|
| 256 |
repo_type="dataset",
|
| 257 |
+
path_in_repo="",
|
| 258 |
+
commit_message=f"sync {ts}",
|
| 259 |
ignore_patterns=UPLOAD_IGNORE,
|
| 260 |
)
|
| 261 |
elapsed = time.time() - t0
|
| 262 |
log(f" β Upload completed ({elapsed:.1f}s)")
|
| 263 |
except Exception as e:
|
| 264 |
elapsed = time.time() - t0
|
| 265 |
+
log(f" β Upload failed ({elapsed:.1f}s): {e}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 266 |
|
| 267 |
log("ββββββββββββββββββββββββββββββββββββββββββββββ")
|
| 268 |
|
ubuntu-server/start-server.sh
CHANGED
|
@@ -2,9 +2,6 @@
|
|
| 2 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 3 |
# HuggingRun Ubuntu Server: ttyd + SSH-over-WebSocket + nginx
|
| 4 |
# Port 7860 (nginx): web terminal + SSH
|
| 5 |
-
#
|
| 6 |
-
# IMPORTANT: nginx starts LAST so all startup output is visible in
|
| 7 |
-
# HF's runtime SSE (which only captures during STARTING state).
|
| 8 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 9 |
|
| 10 |
LOGFILE="/var/log/huggingrun.log"
|
|
@@ -13,12 +10,10 @@ LOGFILE="/var/log/huggingrun.log"
|
|
| 13 |
export SSH_PORT="${SSH_PORT:-2222}"
|
| 14 |
export TTYD_PORT="${TTYD_PORT:-7681}"
|
| 15 |
|
| 16 |
-
#
|
| 17 |
-
# Also tee to logfile for /runlog endpoint
|
| 18 |
-
exec > >(tee -a "$LOGFILE" >&2) 2>&1
|
| 19 |
-
|
| 20 |
log() {
|
| 21 |
-
echo "$*"
|
|
|
|
| 22 |
}
|
| 23 |
|
| 24 |
# ββ Boot info βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -107,19 +102,13 @@ log "[ubuntu] ββ System ready ββ"
|
|
| 107 |
log "[ubuntu] View logs: curl https://<space>.hf.space/runlog"
|
| 108 |
log "[ubuntu] Stream SSE: curl -N https://<space>.hf.space/runlog/stream"
|
| 109 |
|
| 110 |
-
# ββ Heartbeat
|
| 111 |
(while true; do
|
| 112 |
sleep 60
|
| 113 |
log "[ubuntu] heartbeat: $(date -u) | load=$(cat /proc/loadavg 2>/dev/null | cut -d' ' -f1-3) | mem=$(free -h 2>/dev/null | grep Mem: | tr -s ' ' | cut -d' ' -f3,2 | sed 's/ /\//' || echo '?')"
|
| 114 |
done) &
|
| 115 |
|
| 116 |
# ββ Start nginx LAST β opens port 7860 β HF transitions to RUNNING ββ
|
| 117 |
-
# Use /scripts/nginx.conf directly (dataset restore may overwrite /etc/nginx/)
|
| 118 |
log "[ubuntu] nginx config: /scripts/nginx.conf"
|
| 119 |
-
log "[ubuntu] $(head -1 /scripts/nginx.conf 2>&1 || echo 'MISSING')"
|
| 120 |
log "[ubuntu] Starting nginx on 0.0.0.0:7860 (HF will mark RUNNING) ..."
|
| 121 |
-
nginx -c /scripts/nginx.conf -g 'daemon off;'
|
| 122 |
-
NGINX_PID=$!
|
| 123 |
-
log "[ubuntu] nginx started PID=${NGINX_PID}"
|
| 124 |
-
trap "kill $NGINX_PID 2>/dev/null; wait $NGINX_PID 2>/dev/null; exit" SIGTERM SIGINT SIGQUIT
|
| 125 |
-
wait $NGINX_PID
|
|
|
|
| 2 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 3 |
# HuggingRun Ubuntu Server: ttyd + SSH-over-WebSocket + nginx
|
| 4 |
# Port 7860 (nginx): web terminal + SSH
|
|
|
|
|
|
|
|
|
|
| 5 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 6 |
|
| 7 |
LOGFILE="/var/log/huggingrun.log"
|
|
|
|
| 10 |
export SSH_PORT="${SSH_PORT:-2222}"
|
| 11 |
export TTYD_PORT="${TTYD_PORT:-7681}"
|
| 12 |
|
| 13 |
+
# Log to stderr (HF captures stderr) + logfile (for /runlog endpoint)
|
|
|
|
|
|
|
|
|
|
| 14 |
log() {
|
| 15 |
+
echo "$*" >&2
|
| 16 |
+
echo "$*" >> "$LOGFILE"
|
| 17 |
}
|
| 18 |
|
| 19 |
# ββ Boot info βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 102 |
log "[ubuntu] View logs: curl https://<space>.hf.space/runlog"
|
| 103 |
log "[ubuntu] Stream SSE: curl -N https://<space>.hf.space/runlog/stream"
|
| 104 |
|
| 105 |
+
# ββ Heartbeat βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 106 |
(while true; do
|
| 107 |
sleep 60
|
| 108 |
log "[ubuntu] heartbeat: $(date -u) | load=$(cat /proc/loadavg 2>/dev/null | cut -d' ' -f1-3) | mem=$(free -h 2>/dev/null | grep Mem: | tr -s ' ' | cut -d' ' -f3,2 | sed 's/ /\//' || echo '?')"
|
| 109 |
done) &
|
| 110 |
|
| 111 |
# ββ Start nginx LAST β opens port 7860 β HF transitions to RUNNING ββ
|
|
|
|
| 112 |
log "[ubuntu] nginx config: /scripts/nginx.conf"
|
|
|
|
| 113 |
log "[ubuntu] Starting nginx on 0.0.0.0:7860 (HF will mark RUNNING) ..."
|
| 114 |
+
exec nginx -c /scripts/nginx.conf -g 'daemon off;'
|
|
|
|
|
|
|
|
|
|
|
|