feat: add /logs endpoint for reliable log viewing

HF's runtime log API doesn't capture Docker SDK container output.
Added /var/log/huggingrun.log file that all scripts write to,
served via nginx at /logs endpoint. Logs are now viewable with:
curl https://<space>.hf.space/logs

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

scripts/entrypoint.sh

@@ -6,25 +6,32 @@
 # 3. Start sync-loop daemon in BACKGROUND
 # ─────────────────────────────────────────────────────────────────────
 set -e
-# HF Spaces log API captures stderr — redirect ALL stdout → stderr
-exec 1>&2
-echo "========================================"
-echo "[entrypoint] HuggingRun starting ..."
-echo "[entrypoint] Date: $(date -u)"
-echo "[entrypoint] SPACE_ID=${SPACE_ID:-not set}"
-echo "[entrypoint] HF_TOKEN=${HF_TOKEN:+set (${#HF_TOKEN} chars)}${HF_TOKEN:-NOT SET}"
-echo "[entrypoint] HF_DATASET_REPO=${HF_DATASET_REPO:-not set}"
-echo "[entrypoint] PERSIST_PATH=${PERSIST_PATH:-/data}"
-echo "[entrypoint] RUN_CMD=${RUN_CMD:-default}"
-echo "========================================"
+
+LOGFILE="/var/log/huggingrun.log"
+touch "$LOGFILE"
+
+log() {
+    echo "$@" >> "$LOGFILE"
+    echo "$@" >&2
+}
+
+log "========================================"
+log "[entrypoint] HuggingRun starting ..."
+log "[entrypoint] Date: $(date -u)"
+log "[entrypoint] SPACE_ID=${SPACE_ID:-not set}"
+log "[entrypoint] HF_TOKEN=${HF_TOKEN:+set (${#HF_TOKEN} chars)}${HF_TOKEN:-NOT SET}"
+log "[entrypoint] HF_DATASET_REPO=${HF_DATASET_REPO:-not set}"
+log "[entrypoint] PERSIST_PATH=${PERSIST_PATH:-/data}"
+log "[entrypoint] RUN_CMD=${RUN_CMD:-default}"
+log "========================================"
 
 # Determine dataset repo
 if [ -z "$HF_DATASET_REPO" ]; then
     if [ -n "$SPACE_ID" ]; then
         export HF_DATASET_REPO="${SPACE_ID}-data"
-        echo "[entrypoint] Auto-set HF_DATASET_REPO=${HF_DATASET_REPO}"
+        log "[entrypoint] Auto-set HF_DATASET_REPO=${HF_DATASET_REPO}"
     elif [ -n "$HF_TOKEN" ]; then
-        echo "[entrypoint] Resolving HF_DATASET_REPO from token ..."
+        log "[entrypoint] Resolving HF_DATASET_REPO from token ..."
         export HF_DATASET_REPO=$(python3 -c "
 from huggingface_hub import HfApi
 import os
@@ -34,13 +41,13 @@ try:
 except:
     print('')
 " 2>/dev/null)
-        echo "[entrypoint] Resolved: ${HF_DATASET_REPO}"
+        log "[entrypoint] Resolved: ${HF_DATASET_REPO}"
     fi
 fi
 
 # Ensure dataset repo exists
 if [ -n "$HF_TOKEN" ] && [ -n "$HF_DATASET_REPO" ]; then
-    echo "[entrypoint] Verifying dataset: ${HF_DATASET_REPO} ..."
+    log "[entrypoint] Verifying dataset: ${HF_DATASET_REPO} ..."
     python3 -c "
 from huggingface_hub import HfApi
 import os
@@ -52,9 +59,9 @@ try:
 except:
     api.create_repo(repo_id=repo, repo_type='dataset', private=True)
     print(f'[entrypoint] Created dataset: {repo}', flush=True)
-" 2>&1 || echo "[entrypoint] WARNING: Could not verify dataset"
+" >> "$LOGFILE" 2>&1 || log "[entrypoint] WARNING: Could not verify dataset"
 else
-    echo "[entrypoint] WARNING: persistence disabled (no token/repo)"
+    log "[entrypoint] WARNING: persistence disabled (no token/repo)"
 fi
 
 # Write env for other processes
@@ -63,21 +70,21 @@ export HF_TOKEN="${HF_TOKEN}"
 export HF_DATASET_REPO="${HF_DATASET_REPO}"
 export PERSIST_PATH="${PERSIST_PATH:-/data}"
 ENVEOF
-echo "[entrypoint] Wrote /etc/huggingrun.env"
+log "[entrypoint] Wrote /etc/huggingrun.env"
 
 # Background: init (download + restore) then start sync-loop
 # This runs AFTER services start so HF doesn't timeout
 (
-    echo "[entrypoint:bg] Starting init in background ..."
-    python3 -u /opt/git_sync_daemon.py init
-    echo "[entrypoint:bg] Init done, starting sync-loop ..."
-    exec python3 -u /opt/git_sync_daemon.py sync-loop
+    log "[entrypoint:bg] Starting init in background ..."
+    python3 -u /opt/git_sync_daemon.py init >> "$LOGFILE" 2>&1
+    log "[entrypoint:bg] Init done, starting sync-loop ..."
+    exec python3 -u /opt/git_sync_daemon.py sync-loop >> "$LOGFILE" 2>&1
 ) &
 BG_PID=$!
-echo "[entrypoint] Background init+sync PID=${BG_PID}"
+log "[entrypoint] Background init+sync PID=${BG_PID}"
 
 # Start services immediately (so HF sees port 7860)
 CMD="${RUN_CMD:-python3 /app/demo_app.py}"
-echo "[entrypoint] Starting services: ${CMD}"
-echo "========================================"
+log "[entrypoint] Starting services: ${CMD}"
+log "========================================"
 exec $CMD

ubuntu-server/nginx.conf

@@ -33,6 +33,13 @@ http {
             proxy_send_timeout 86400;
         }
 
+        # /logs → live container logs
+        location /logs {
+            default_type text/plain;
+            add_header Cache-Control "no-cache, no-store";
+            alias /var/log/huggingrun.log;
+        }
+
         # Everything else → ttyd web terminal (on 7681)
         location / {
             proxy_pass http://127.0.0.1:7681;

ubuntu-server/start-server.sh

@@ -4,29 +4,27 @@
 # Port 7860 (nginx): web terminal + SSH
 # ─────────────────────────────────────────────────────────────────────
 
+LOGFILE="/var/log/huggingrun.log"
+> "$LOGFILE"  # truncate on start
+
 export SSH_PORT="${SSH_PORT:-2222}"
 export TTYD_PORT="${TTYD_PORT:-7681}"
 
-# Helper: write to BOTH stdout and stderr, and also /proc/1/fd/1 + /proc/1/fd/2
-# This ensures HF captures the output regardless of which stream they monitor
+# Log to file + stderr (HF may or may not capture stderr)
 log() {
-    echo "$@"
-    echo "$@" >&2
-    echo "$@" > /proc/1/fd/1 2>/dev/null || true
-    echo "$@" > /proc/1/fd/2 2>/dev/null || true
+    local msg="$*"
+    echo "$msg" >> "$LOGFILE"
+    echo "$msg" >&2
 }
 
-# ── Start nginx FIRST so HF sees port 7860 and starts capturing logs ──
+# ── Start nginx FIRST so HF sees port 7860 ──────────────────────
 nginx -g 'daemon off;' &
 NGINX_PID=$!
-
-# Forward signals to nginx for clean shutdown
 trap "kill $NGINX_PID 2>/dev/null; wait $NGINX_PID 2>/dev/null; exit" SIGTERM SIGINT SIGQUIT
 
-# Small delay to let nginx bind and HF detect the port
-sleep 3
+sleep 2
 
-# ── Now print boot info (HF should be streaming logs at this point) ──
+# ── Boot info ─────────────────────────────────────────────────────
 log "========================================"
 log "[ubuntu] HuggingRun Ubuntu Server"
 log "[ubuntu] $(date -u)"
@@ -39,9 +37,9 @@ log "[ubuntu] Disk: $(df -h / 2>/dev/null | awk 'NR==2{print $2, "total,", $4, "
 log "[ubuntu] User: $(whoami) (uid=$(id -u))"
 log "========================================"
 
-# ── Network info ───────────────────────────────────────────────────
+# ── Network info ──────────────────────────────────────────────────
 log "[ubuntu] Network interfaces:"
-ip -4 addr show 2>/dev/null | grep -E 'inet |^[0-9]' | while read line; do log "  $line"; done || true
+ip -4 addr show 2>/dev/null | grep -E 'inet |^[0-9]' | while IFS= read -r line; do log "  $line"; done
 
 # ── sshd ──────────────────────────────────────────────────────────
 mkdir -p /run/sshd
@@ -61,7 +59,7 @@ else
     log "[ubuntu] [FAILED] sshd failed to start"
 fi
 
-# ── WebSocket-to-SSH bridge ──────────────────────────────────────
+# ── WebSocket-to-SSH bridge ───────────────────────────────────────
 log "[ubuntu] Starting WS-SSH bridge on 127.0.0.1:7862 ..."
 python3 /opt/ws-ssh-bridge.py &
 BRIDGE_PID=$!
@@ -72,7 +70,7 @@ else
     log "[ubuntu] [FAILED] WS-SSH bridge failed to start"
 fi
 
-# ── ttyd (web terminal) ─────────────────────────────────────────
+# ── ttyd (web terminal) ──────────────────────────────────────────
 log "[ubuntu] Starting ttyd on 127.0.0.1:${TTYD_PORT} ..."
 ttyd --port "$TTYD_PORT" --writable --base-path / bash --login &
 TTYD_PID=$!
@@ -83,7 +81,7 @@ else
     log "[ubuntu] [FAILED] ttyd failed to start"
 fi
 
-# ── Process summary ──────────────────────────────────────────────
+# ── Process summary ───────────────────────────────────────────────
 log "========================================"
 log "[ubuntu] Services:"
 log "[ubuntu]   nginx         PID=${NGINX_PID}   0.0.0.0:7860"
@@ -91,26 +89,20 @@ log "[ubuntu]   sshd          PID=${SSHD_PID}   127.0.0.1:${SSH_PORT}"
 log "[ubuntu]   ws-ssh-bridge PID=${BRIDGE_PID}  127.0.0.1:7862"
 log "[ubuntu]   ttyd          PID=${TTYD_PID}    127.0.0.1:${TTYD_PORT}"
 log "========================================"
-
-# ── Installed packages ───────────────────────────────────────────
 log "[ubuntu] Base packages: $(wc -l < /etc/base-packages.list 2>/dev/null || echo '?')"
 log "[ubuntu] Current packages: $(dpkg-query -W -f='\n' 2>/dev/null | wc -l)"
 
-# ── Running processes ────────────────────────────────────────────
 log "[ubuntu] All processes:"
-ps aux --no-headers 2>/dev/null | awk '{printf "[ubuntu]   %-8s PID=%-6s %s\n", $1, $2, $11}' | while read line; do log "$line"; done || true
+ps aux --no-headers 2>/dev/null | awk '{printf "[ubuntu]   %-8s PID=%-6s %s\n", $1, $2, $11}' | while IFS= read -r line; do log "$line"; done
 
 log "[ubuntu] ══ System ready ══"
+log "[ubuntu] View logs: curl https://<space>.hf.space/logs"
 
-# ── Heartbeat (every 10s for debugging, will increase later) ─────
+# ── Heartbeat ─────────────────────────────────────────────────────
 (while true; do
-    sleep 10
-    MSG="[ubuntu] heartbeat: $(date -u) | load=$(cat /proc/loadavg 2>/dev/null | cut -d' ' -f1-3)"
-    echo "$MSG"
-    echo "$MSG" >&2
-    echo "$MSG" > /proc/1/fd/1 2>/dev/null || true
-    echo "$MSG" > /proc/1/fd/2 2>/dev/null || true
+    sleep 60
+    log "[ubuntu] heartbeat: $(date -u) | load=$(cat /proc/loadavg 2>/dev/null | cut -d' ' -f1-3) | mem=$(free -h 2>/dev/null | awk '/Mem:/{print $3"/"$2}' || echo '?')"
 done) &
 
-# ── Wait for nginx (keep shell as PID 1 for signal handling) ─────
+# ── Wait for nginx ────────────────────────────────────────────────
 wait $NGINX_PID