tao-shen Claude Opus 4.6 commited on
Commit
ea03e2a
Β·
1 Parent(s): bfc7c5f

fix: restore original logging pattern + fix sync hanging

Browse files

Root cause analysis:
1. sync daemon log() changed from stderr to stdout (broke HF log capture)
2. stdbuf -oL -eL removed from RUN_CMD (broke line buffering)
3. start-server.sh used exec > >(tee >&2) instead of direct echo >&2
4. upload_large_folder hangs for hours on 1.8GB rootfs

Fixes:
- Restore log() to use file=sys.stderr (original design from ea21d45)
- Restore stdbuf -oL -eL in Dockerfile RUN_CMD
- Simplify start-server.sh to direct echo >&2 + logfile write
- Switch from upload_large_folder to upload_folder (doesn't hang)
- Restore exec nginx (original PID 1 = nginx, not bash+wait)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Dockerfile CHANGED
@@ -55,7 +55,7 @@ RUN chmod +x /scripts/entrypoint.sh /scripts/start-server.sh
55
 
56
  ENV PERSIST_PATH=/data
57
  ENV PYTHONUNBUFFERED=1
58
- ENV RUN_CMD="/scripts/start-server.sh"
59
  ENV SSH_PORT=2222
60
 
61
  # Run as root (needed for: apt install persistence, bind mounts, sshd)
 
55
 
56
  ENV PERSIST_PATH=/data
57
  ENV PYTHONUNBUFFERED=1
58
+ ENV RUN_CMD="stdbuf -oL -eL /scripts/start-server.sh"
59
  ENV SSH_PORT=2222
60
 
61
  # Run as root (needed for: apt install persistence, bind mounts, sshd)
ubuntu-server/git_sync_daemon.py CHANGED
@@ -18,6 +18,7 @@ import time
18
  import subprocess
19
  import signal
20
  import shutil
 
21
 
22
  # ── Config ─────────────────────────────────────────────────────────
23
  PERSIST_PATH = os.environ.get("PERSIST_PATH", "/data")
@@ -57,7 +58,7 @@ LOGFILE = "/var/log/huggingrun.log"
57
  def log(msg):
58
  ts = time.strftime("%H:%M:%S", time.gmtime())
59
  line = f"[sync {ts}] {msg}"
60
- print(line, flush=True)
61
  try:
62
  with open(LOGFILE, "a") as f:
63
  f.write(line + "\n")
@@ -244,38 +245,24 @@ def save_and_upload():
244
  log("── UPLOAD: /data β†’ dataset")
245
  api = HfApi(token=HF_TOKEN)
246
  ts = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
247
- log(f" repo={HF_DATASET_REPO}, method=upload_large_folder")
248
  log(f" ignore={UPLOAD_IGNORE}")
249
 
250
  t0 = time.time()
251
  try:
252
- api.upload_large_folder(
253
  folder_path=PERSIST_PATH,
254
  repo_id=HF_DATASET_REPO,
255
  repo_type="dataset",
 
 
256
  ignore_patterns=UPLOAD_IGNORE,
257
  )
258
  elapsed = time.time() - t0
259
  log(f" βœ“ Upload completed ({elapsed:.1f}s)")
260
  except Exception as e:
261
  elapsed = time.time() - t0
262
- log(f" βœ— upload_large_folder failed ({elapsed:.1f}s): {e}")
263
- log(" Trying fallback: upload_folder ...")
264
- t0 = time.time()
265
- try:
266
- api.upload_folder(
267
- folder_path=PERSIST_PATH,
268
- repo_id=HF_DATASET_REPO,
269
- repo_type="dataset",
270
- path_in_repo="",
271
- commit_message=f"sync {ts}",
272
- ignore_patterns=UPLOAD_IGNORE,
273
- )
274
- elapsed = time.time() - t0
275
- log(f" βœ“ Upload completed via fallback ({elapsed:.1f}s)")
276
- except Exception as e2:
277
- elapsed = time.time() - t0
278
- log(f" βœ— Fallback also failed ({elapsed:.1f}s): {e2}")
279
 
280
  log("══════════════════════════════════════════════")
281
 
 
18
  import subprocess
19
  import signal
20
  import shutil
21
+ import threading
22
 
23
  # ── Config ─────────────────────────────────────────────────────────
24
  PERSIST_PATH = os.environ.get("PERSIST_PATH", "/data")
 
58
  def log(msg):
59
  ts = time.strftime("%H:%M:%S", time.gmtime())
60
  line = f"[sync {ts}] {msg}"
61
+ print(line, file=sys.stderr, flush=True)
62
  try:
63
  with open(LOGFILE, "a") as f:
64
  f.write(line + "\n")
 
245
  log("── UPLOAD: /data β†’ dataset")
246
  api = HfApi(token=HF_TOKEN)
247
  ts = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
248
+ log(f" repo={HF_DATASET_REPO}, method=upload_folder")
249
  log(f" ignore={UPLOAD_IGNORE}")
250
 
251
  t0 = time.time()
252
  try:
253
+ api.upload_folder(
254
  folder_path=PERSIST_PATH,
255
  repo_id=HF_DATASET_REPO,
256
  repo_type="dataset",
257
+ path_in_repo="",
258
+ commit_message=f"sync {ts}",
259
  ignore_patterns=UPLOAD_IGNORE,
260
  )
261
  elapsed = time.time() - t0
262
  log(f" βœ“ Upload completed ({elapsed:.1f}s)")
263
  except Exception as e:
264
  elapsed = time.time() - t0
265
+ log(f" βœ— Upload failed ({elapsed:.1f}s): {e}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
266
 
267
  log("══════════════════════════════════════════════")
268
 
ubuntu-server/start-server.sh CHANGED
@@ -2,9 +2,6 @@
2
  # ─────────────────────────────────────────────────────────────────────
3
  # HuggingRun Ubuntu Server: ttyd + SSH-over-WebSocket + nginx
4
  # Port 7860 (nginx): web terminal + SSH
5
- #
6
- # IMPORTANT: nginx starts LAST so all startup output is visible in
7
- # HF's runtime SSE (which only captures during STARTING state).
8
  # ─────────────────────────────────────────────────────────────────────
9
 
10
  LOGFILE="/var/log/huggingrun.log"
@@ -13,12 +10,10 @@ LOGFILE="/var/log/huggingrun.log"
13
  export SSH_PORT="${SSH_PORT:-2222}"
14
  export TTYD_PORT="${TTYD_PORT:-7681}"
15
 
16
- # Redirect ALL stdout to stderr (HF captures stderr for Docker SDK spaces)
17
- # Also tee to logfile for /runlog endpoint
18
- exec > >(tee -a "$LOGFILE" >&2) 2>&1
19
-
20
  log() {
21
- echo "$*"
 
22
  }
23
 
24
  # ── Boot info ─────────────────────────────────────────────────────
@@ -107,19 +102,13 @@ log "[ubuntu] ══ System ready ══"
107
  log "[ubuntu] View logs: curl https://<space>.hf.space/runlog"
108
  log "[ubuntu] Stream SSE: curl -N https://<space>.hf.space/runlog/stream"
109
 
110
- # ── Heartbeat (writes to logfile for /api/logs) ──────────────────
111
  (while true; do
112
  sleep 60
113
  log "[ubuntu] heartbeat: $(date -u) | load=$(cat /proc/loadavg 2>/dev/null | cut -d' ' -f1-3) | mem=$(free -h 2>/dev/null | grep Mem: | tr -s ' ' | cut -d' ' -f3,2 | sed 's/ /\//' || echo '?')"
114
  done) &
115
 
116
  # ── Start nginx LAST β†’ opens port 7860 β†’ HF transitions to RUNNING ──
117
- # Use /scripts/nginx.conf directly (dataset restore may overwrite /etc/nginx/)
118
  log "[ubuntu] nginx config: /scripts/nginx.conf"
119
- log "[ubuntu] $(head -1 /scripts/nginx.conf 2>&1 || echo 'MISSING')"
120
  log "[ubuntu] Starting nginx on 0.0.0.0:7860 (HF will mark RUNNING) ..."
121
- nginx -c /scripts/nginx.conf -g 'daemon off;' &
122
- NGINX_PID=$!
123
- log "[ubuntu] nginx started PID=${NGINX_PID}"
124
- trap "kill $NGINX_PID 2>/dev/null; wait $NGINX_PID 2>/dev/null; exit" SIGTERM SIGINT SIGQUIT
125
- wait $NGINX_PID
 
2
  # ─────────────────────────────────────────────────────────────────────
3
  # HuggingRun Ubuntu Server: ttyd + SSH-over-WebSocket + nginx
4
  # Port 7860 (nginx): web terminal + SSH
 
 
 
5
  # ─────────────────────────────────────────────────────────────────────
6
 
7
  LOGFILE="/var/log/huggingrun.log"
 
10
  export SSH_PORT="${SSH_PORT:-2222}"
11
  export TTYD_PORT="${TTYD_PORT:-7681}"
12
 
13
+ # Log to stderr (HF captures stderr) + logfile (for /runlog endpoint)
 
 
 
14
  log() {
15
+ echo "$*" >&2
16
+ echo "$*" >> "$LOGFILE"
17
  }
18
 
19
  # ── Boot info ─────────────────────────────────────────────────────
 
102
  log "[ubuntu] View logs: curl https://<space>.hf.space/runlog"
103
  log "[ubuntu] Stream SSE: curl -N https://<space>.hf.space/runlog/stream"
104
 
105
+ # ── Heartbeat ─────────────────────────────────────────────────────
106
  (while true; do
107
  sleep 60
108
  log "[ubuntu] heartbeat: $(date -u) | load=$(cat /proc/loadavg 2>/dev/null | cut -d' ' -f1-3) | mem=$(free -h 2>/dev/null | grep Mem: | tr -s ' ' | cut -d' ' -f3,2 | sed 's/ /\//' || echo '?')"
109
  done) &
110
 
111
  # ── Start nginx LAST β†’ opens port 7860 β†’ HF transitions to RUNNING ──
 
112
  log "[ubuntu] nginx config: /scripts/nginx.conf"
 
113
  log "[ubuntu] Starting nginx on 0.0.0.0:7860 (HF will mark RUNNING) ..."
114
+ exec nginx -c /scripts/nginx.conf -g 'daemon off;'