Spaces:

SZFSPC
/

apple

Paused

apple / scripts /hf-entrypoint.sh
fix: improve SSH service stability and backup.py error handling
1d709b5 verified 5 months ago
36.4 kB
#!/usr/bin/env bash
# hf-entrypoint.sh - HF Spaces 容器入口
set -euo pipefail

# ---- Force line-buffered stdout/stderr for docker logs ----
# In Docker, stdout goes through a pipe and is FULLY buffered (4KB+).
# If we exit 1 before the buffer fills, output is LOST.
# Re-exec ourselves through stdbuf -oL -eL to force line buffering.
if [[ -z "${_ENTRYPOINT_UNBUFFERED:-}" ]]; then
    export _ENTRYPOINT_UNBUFFERED=1
    if command -v stdbuf >/dev/null 2>&1; then
        # Best case: stdbuf is available, re-exec with line-buffered I/O
        exec stdbuf -oL -eL "$0" "$@"
    else
        # Fallback: pipe ourselves through `cat` (which is line-buffered)
        exec "$@" 2>&1 | cat
    fi
fi

# ---- Fix terminal for interactive input ----
if [[ -t 0 ]]; then
    stty sane 2>/dev/null || true
fi

echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: starting..."

# ============================================
# 0. 保存 HF Space 注入的环境变量（优先级最高）
# ============================================
declare -A _HF_SPACE_ENV
while IFS='=' read -r _key _value; do
  _HF_SPACE_ENV["$_key"]="$_value"
done < <(env)
unset _key _value

# 保存环境变量到 /etc/profile.d，供后续 bash 会话使用
if [[ -x /usr/local/bin/save-env.sh ]]; then
    /usr/local/bin/save-env.sh
else
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: warning: save-env.sh not found, skipping env export"
fi

# 加载已保存的环境变量
if [[ -f /etc/profile.d/openclaw-env.sh ]]; then
    # shellcheck source=/dev/null
    source /etc/profile.d/openclaw-env.sh
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: loaded environment from /etc/profile.d/openclaw-env.sh"
fi

# 恢复 HF Space 注入的环境变量（确保 HF Space 优先级最高）
for _var in "${!_HF_SPACE_ENV[@]}"; do
  export "$_var"="${_HF_SPACE_ENV[$_var]}"
done
unset _HF_SPACE_ENV _var

# ============================================
# 0.5 运行时升级 OpenClaw（如果指定了 OPENCLAW_VERSION 且不是镜像构建版本）
# ============================================
TARGET_OPENCLAW_VERSION="${OPENCLAW_VERSION:-}"
if [[ -n "$TARGET_OPENCLAW_VERSION" ]] && [[ "$TARGET_OPENCLAW_VERSION" != "latest" ]]; then
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: checking OpenClaw version upgrade..."
    
    if [[ -x /usr/local/bin/upgrade-openclaw.sh ]]; then
        CURRENT_VERSION="$(openclaw --version 2>/dev/null | head -1 | tr -d '[:space:]')" || CURRENT_VERSION="unknown"
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: current OpenClaw version: ${CURRENT_VERSION}"
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: target OpenClaw version: ${TARGET_OPENCLAW_VERSION}"
        
        if [[ "$CURRENT_VERSION" != "$TARGET_OPENCLAW_VERSION" ]] && [[ "$CURRENT_VERSION" != *"$TARGET_OPENCLAW_VERSION"* ]]; then
            echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: upgrading OpenClaw from ${CURRENT_VERSION} to ${TARGET_OPENCLAW_VERSION}..."
            /usr/local/bin/upgrade-openclaw.sh --version "$TARGET_OPENCLAW_VERSION" --skip-backup || {
                echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: WARNING: OpenClaw upgrade failed, continuing with current version"
            }
            echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: OpenClaw upgrade completed"
        else
            echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: OpenClaw is already at target version, skipping upgrade"
        fi
    else
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: WARNING: upgrade-openclaw.sh not found, skipping runtime upgrade"
    fi
elif [[ "$TARGET_OPENCLAW_VERSION" == "latest" ]]; then
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: OPENCLAW_VERSION=latest, will check for updates..."
    
    if [[ -x /usr/local/bin/upgrade-openclaw.sh ]]; then
        /usr/local/bin/upgrade-openclaw.sh --latest --skip-backup || {
            echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: WARNING: OpenClaw latest check failed, continuing with current version"
        }
    fi
else
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: OPENCLAW_VERSION not set, using image built-in version"
fi

# ============================================
# 1. 启动 supervisord（管理 cron + openclaw-gateway）
# ============================================
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: starting supervisord..."
mkdir -p /var/run /var/log/supervisor /var/log/hf-entrypoint
touch /var/log/hf-entrypoint/supervisord-stdout.log /var/log/hf-entrypoint/supervisord-stderr.log
/usr/bin/supervisord -c /etc/supervisor/supervisord.conf \
    > >(tee -a /var/log/hf-entrypoint/supervisord-stdout.log) \
    2> >(tee -a /var/log/hf-entrypoint/supervisord-stderr.log >&2) &
SUPERVISORD_PID=$!
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: supervisord started (pid=$SUPERVISORD_PID)"

# 启动后台 tail -f 进程，把 openclaw-gateway 的日志实时转发到容器 stdout/stderr
# 这样用户在 docker logs 中可以看到 restore 的所有输出
touch /var/log/hf-entrypoint/openclaw-gateway-stdout.log /var/log/hf-entrypoint/openclaw-gateway-stderr.log
if command -v tail >/dev/null 2>&1; then
    # stdout: prefix with [gateway-out so users can distinguish
    tail -f /var/log/hf-entrypoint/openclaw-gateway-stdout.log 2>/dev/null &
    TAIL_STDOUT_PID=$!
    # stderr: prefix with [gateway-err]
    tail -f /var/log/hf-entrypoint/openclaw-gateway-stderr.log 2>/dev/null >&2 &
    TAIL_STDERR_PID=$!
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: log forwarders started (stdout_pid=$TAIL_STDOUT_PID, stderr_pid=$TAIL_STDERR_PID)"
fi

while [[ ! -f /var/run/supervisord.pid ]]; do
    sleep 0.5
done
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: supervisord is ready"

# 1.0 启动 SSH 服务和看门狗（确保SSH持续可用）
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: starting SSH service and watchdog..."

# 0. 设置root密码（如果已设置ROOT_PASSWORD环境变量）
if [ -n "${ROOT_PASSWORD:-}" ]; then
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: setting root password..."
    echo "root:${ROOT_PASSWORD}" | chpasswd 2>/dev/null
    if [ $? -eq 0 ]; then
        # 确保root账户未锁定
        passwd -u root 2>/dev/null || true
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: root password set successfully"
    else
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: WARNING: failed to set root password"
    fi
fi

# 0.1 创建 SSH 权限分离目录并清理残留 PID/套接字
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: preparing SSH runtime directories..."
mkdir -p /run/sshd /var/run/sshd 2>/dev/null || true
chmod 755 /run/sshd /var/run/sshd 2>/dev/null || true
rm -f /var/run/sshd.pid /var/run/sshd.init.pid /tmp/ssh-* 2>/dev/null || true

# 0.2 生成 SSH 主机密钥（如果不存在）
if [ ! -f "/etc/ssh/ssh_host_rsa_key" ]; then
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: generating SSH host keys..."
    ssh-keygen -A 2>/dev/null || true
fi

# 1. 确保SSH服务启动
if ! pgrep -x "sshd" > /dev/null 2>&1; then
    _sshd_bin=""
    if [ -x "/usr/sbin/sshd" ]; then
        _sshd_bin="/usr/sbin/sshd"
    elif [ -x "/usr/bin/sshd" ]; then
        _sshd_bin="/usr/bin/sshd"
    fi
    
    if [ -n "$_sshd_bin" ]; then
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: starting sshd from $_sshd_bin..."
        $_sshd_bin
        sleep 2
        if pgrep -x "sshd" > /dev/null 2>&1; then
            echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: sshd started successfully"
            # 保护 sshd 不被 OOM killer 杀死（降低优先级）
            for _pid in $(pgrep -x "sshd" 2>/dev/null); do
                echo -500 > /proc/$_pid/oom_score_adj 2>/dev/null || true
            done
            echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: sshd OOM protection applied (oom_score_adj=-500)"
            # 调整内核内存策略，降低 OOM 误杀关键服务的概率（容器内 /proc 通常只读，失败仅警告）
            if echo "2" > /proc/sys/vm/overcommit_memory 2>/dev/null; then
                echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: vm.overcommit_memory=2 set (never overcommit)"
            else
                echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: WARNING: vm.overcommit_memory could not be set (/proc is read-only in container, using host defaults)"
            fi
        else
            echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: WARNING: sshd failed to start, will be handled by watchdog"
        fi
    else
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: WARNING: sshd executable not found"
    fi
fi

# 启动SSH看门狗（确保SSH服务持续可用）
# 环境变量 SSH_WATCHDOG_DOCKER_LOG: 控制看门狗日志是否重定向到 Docker logs（默认 true）
SSH_WATCHDOG_DOCKER_LOG="${SSH_WATCHDOG_DOCKER_LOG:-false}"
if [ -x "/usr/local/bin/ssh_service_watchdog.sh" ]; then
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: starting SSH watchdog (docker_log=$SSH_WATCHDOG_DOCKER_LOG)..."
    if [ "$SSH_WATCHDOG_DOCKER_LOG" = "true" ]; then
        # 将看门狗输出重定向到Docker标准输出，这样即使xterm无法连接，也可以通过docker logs查看
        nohup /usr/local/bin/ssh_service_watchdog.sh >> /proc/1/fd/1 2>> /proc/1/fd/2 &
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: SSH watchdog logs are available via: docker logs <container>"
    else
        # 不重定向到 Docker logs，日志仅写入本地文件 /var/log/ssh_watchdog.log
        nohup /usr/local/bin/ssh_service_watchdog.sh > /dev/null 2>&1 &
    fi
    SSH_WATCHDOG_PID=$!
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: SSH watchdog started (pid=$SSH_WATCHDOG_PID)"
else
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: WARNING: ssh_service_watchdog.sh not found"
fi

# 1.1 启动 BT Panel（与 restore 并行启动，节省时间）
# ============================================
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: starting BT Panel..."
if [[ -f "/www/server/panel/default.pl" ]]; then
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: BT Panel is installed, starting..."
    bt start 2>/dev/null || true
    bt default 2>/dev/null || true
else
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: BT Panel not installed, skipping"
fi

# ============================================
# 0.9 辅助函数：日志转储（在所有失败/关键路径上调用）
# ============================================
# 日志文件路径（必须在函数之前定义，也用于 restore 循环）
RESTORE_LOG_FILE="/var/log/openclaw/restore.log"
OPENCLAW_STDERR_LOG="/var/log/hf-entrypoint/openclaw-gateway-stderr.log"
OPENCLAW_STDOUT_LOG="/var/log/hf-entrypoint/openclaw-gateway-stdout.log"

_dump_log_file() {
    # $1: label
    # $2: log file path
    # $3: max lines (default 500; 0 = full dump)
    # FAST PATH: use cat/tail directly; never pipe through bash while-read.
    local label="$1"
    local logfile="$2"
    local max_lines="${3:-500}"

    echo ""
    echo "========================================================================"
    echo " LOG: ${label}"
    echo " File: ${logfile}"
    echo "========================================================================"

    if [[ ! -f "$logfile" ]]; then
        echo " (file not found)"
        echo "========================================================================"
        return
    fi

    local total_lines
    total_lines=$(wc -l < "$logfile" 2>/dev/null || echo "0")
    local file_size
    file_size=$(stat -c%s "$logfile" 2>/dev/null || echo "0")

    if [[ "$total_lines" -eq 0 ]] || [[ "$file_size" -eq 0 ]]; then
        echo " (file is empty)"
        echo "========================================================================"
        return
    fi

    echo " Lines: ${total_lines}   Size: ${file_size} bytes"
    echo "------------------------------------------------------------------------"

    # CRITICAL: use cat/tail directly. Do NOT pipe through bash while-read.
    # A while-read loop on a 1000-line file can take 2-5 seconds; if the
    # container is killed during that time, output is LOST.
    if [[ "$max_lines" == "0" ]]; then
        cat "$logfile"
    elif [[ "$total_lines" -gt "$max_lines" ]]; then
        echo " (showing LAST ${max_lines} lines of ${total_lines})"
        echo "------------------------------------------------------------------------"
        tail -n "$max_lines" "$logfile"
    else
        cat "$logfile"
    fi
    echo "========================================================================"
}


_dump_restore_logs() {
    # $1: scenario description
    # FAST PATH: direct cat to stdout, NO file persistence (HF wipes files on restart)
    # Output is LINE-BUFFERED thanks to stdbuf re-exec at top of script
    local scenario="${1:-restore ended}"

    echo ""
    echo "========================================================================"
    echo "  RESTORE FAILURE DIAGNOSTIC — ${scenario}"
    echo "  Time: $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
    echo "========================================================================"

    # --- 1. restore.log: the single most important file ---
    echo ""
    echo ">>>> 1. RESTORE LOG (backup.py restore output) <<<<" 
    if [[ -f "$RESTORE_LOG_FILE" ]]; then
        echo "  File: $RESTORE_LOG_FILE"
        echo "  Size: $(stat -c%s "$RESTORE_LOG_FILE" 2>/dev/null || echo unknown) bytes"
        echo "  Lines: $(wc -l < "$RESTORE_LOG_FILE" 2>/dev/null || echo unknown)"
        echo "------------------------------------------------------------------------"
        # FAST: single cat call. Last 300 lines — enough to see error context,
        # small enough to be captured by docker logs before exit 1.
        tail -n 300 "$RESTORE_LOG_FILE"
    else
        echo "  (file not found: $RESTORE_LOG_FILE)"
    fi

    # --- 2. gateway stderr: Python tracebacks live here ---
    echo ""
    echo ">>>> 2. GATEWAY STDERR (Python tracebacks / crashes) <<<<" 
    if [[ -f "$OPENCLAW_STDERR_LOG" ]]; then
        echo "  File: $OPENCLAW_STDERR_LOG"
        echo "  Size: $(stat -c%s "$OPENCLAW_STDERR_LOG" 2>/dev/null || echo unknown) bytes"
        echo "------------------------------------------------------------------------"
        tail -n 150 "$OPENCLAW_STDERR_LOG"
    else
        echo "  (file not found: $OPENCLAW_STDERR_LOG)"
    fi

    # --- 3. gateway stdout ---
    echo ""
    echo ">>>> 3. GATEWAY STDOUT (last 100 lines) <<<<" 
    if [[ -f "$OPENCLAW_STDOUT_LOG" ]]; then
        tail -n 100 "$OPENCLAW_STDOUT_LOG"
    else
        echo "  (file not found: $OPENCLAW_STDOUT_LOG)"
    fi

    # --- 4. Process status (quick) ---
    echo ""
    echo ">>>> 4. RUNNING PROCESSES <<<<" 
    ps auxf 2>/dev/null | head -30 || ps aux 2>/dev/null | head -20 || echo "(ps unavailable)"

    # --- 5. Supervisor status (quick) ---
    echo ""
    echo ">>>> 5. SUPERVISOR STATUS <<<<" 
    if command -v supervisorctl >/dev/null 2>&1; then
        supervisorctl status 2>&1 || echo "(supervisorctl failed)"
    else
        echo "(supervisorctl not available)"
    fi

    # --- 6. Flag files ---
    echo ""
    echo ">>>> 6. STATE FLAGS <<<<" 
    for flag in /root/dataset-restore-failed /root/dataset-restore-completed /root/dataset-restore-fail-count; do
        if [[ -f "$flag" ]]; then
            echo "  $flag: $(cat "$flag" 2>/dev/null)"
        else
            echo "  $flag: (not present)"
        fi
    done

    echo ""
    echo "========================================================================"
    echo "  END DIAGNOSTIC — about to exit 1"
    echo "========================================================================"
    echo ""
}
# ============================================
# 1.2 连续重启保护（电路熔断器）
# ============================================
RESTORE_FAIL_COUNT_FILE="/root/dataset-restore-fail-count"
RESTORE_MAX_CONSECUTIVE_FAILURES=3
RESTORE_COMPLETED_FILE="/root/dataset-restore-completed"

# 检测上一次是否为成功恢复（成功标记存在则判定成功）
if [[ -f "$RESTORE_COMPLETED_FILE" ]]; then
    # 上一次容器是成功恢复的，重置失败计数
    if [[ -f "$RESTORE_FAIL_COUNT_FILE" ]]; then
        rm -f "$RESTORE_FAIL_COUNT_FILE"
    fi
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: Previous restore was successful, resetting failure counter"
else
    # 上次容器可能失败了（未完成恢复），增加失败计数
    mkdir -p "$(dirname "$RESTORE_FAIL_COUNT_FILE")"
    fail_count=0
    if [[ -f "$RESTORE_FAIL_COUNT_FILE" ]]; then
        fail_count=$(cat "$RESTORE_FAIL_COUNT_FILE" 2>/dev/null || echo "0")
    fi
    fail_count=$((fail_count + 1))
    echo "$fail_count" > "$RESTORE_FAIL_COUNT_FILE"
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: Consecutive restore failures: $fail_count / $RESTORE_MAX_CONSECUTIVE_FAILURES"

    if [[ $fail_count -ge $RESTORE_MAX_CONSECUTIVE_FAILURES ]]; then
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ✗ CIRCUIT BREAKER: $fail_count consecutive restore failures detected"
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ✗ Auto-restart disabled. Manual intervention required."
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ✗ To reset: delete $RESTORE_FAIL_COUNT_FILE and restart the Space"
        # 转储完整的恢复日志（不过滤关键字），确保用户看到崩溃前的所有信息
        _dump_restore_logs "circuit breaker triggered after ${fail_count} consecutive failures"
        # ============================================================
        # FLUSHING OUTPUT — DO NOT EXIT PREMATURELY
        # Giving docker runtime 30+ seconds to drain stdout pipe.
        # Progressive FLUSH_MARKER_01..30 are emitted 1s apart —
        # the highest-numbered marker you see in docker logs tells
        # you how much output was captured before the container died.
        # ============================================================
        sync 2>/dev/null || true
        
        echo "========================================================================"
        echo "  ABOUT TO EXIT (code 1) — flushing output to docker logs..."
        echo "  Started: $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
        echo "  Diagnostics above: FULL dump of restore.log + gateway stderr"
        echo "========================================================================"
        sleep 1
        
        for _flush_n in $(seq 1 30); do
            printf -- "-- FLUSH_MARKER_%02d %s --\n" "$_flush_n" "$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
            sleep 1
        done
        
        echo "========================================================================"
        echo "  ALL DIAGNOSTICS FLUSHED TO STDOUT — exiting now"
        echo "  If you see this line and FLUSH_MARKER_30, output was FULLY captured"
        echo "========================================================================"
        sync 2>/dev/null || true
        sleep 1
        exit 1
    fi
fi

# ============================================
# 1.3 等待 openclaw-gateway 完成恢复
# ============================================
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: waiting for openclaw-gateway to complete restore..."

# 注意：RESTORE_COMPLETED_FILE / RESTORE_LOG_FILE / OPENCLAW_STDERR_LOG / OPENCLAW_STDOUT_LOG 已在 0.9 节统一声明
# 持久化恢复标记（/tmp 在 tmpfs 中，容器重启后会丢失；改用 /root/.openclaw/ 确保重启后仍存在）
RESTORE_FAILED_FILE="/root/dataset-restore-failed"
WAITED=0
LAST_RESTORE_LINE=0
LAST_LOG_SIZE=0
PROGRESS_CHECK_INTERVAL=20
SLOW_WARN_THRESHOLD=900   # 15分钟超时预警
MAX_WAIT_TIMEOUT=28800     # 最大等待28800秒（8小时），超时后强制继续启动
IDLE_WARN_THRESHOLD=120   # 日志无新内容120秒则告警
TIME_NO_NEW_LOG=0
gateway_was_seen_alive=false

mkdir -p "$(dirname "$RESTORE_LOG_FILE")"

show_restore_progress() {
    if [[ ! -f "$RESTORE_LOG_FILE" ]]; then
        return
    fi

    local current_lines
    current_lines=$(wc -l < "$RESTORE_LOG_FILE" 2>/dev/null || echo "0")

    if [[ -n "$current_lines" ]] && [[ "$current_lines" -gt "$LAST_RESTORE_LINE" ]]; then
        local new_lines=$((current_lines - LAST_RESTORE_LINE))
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: --- Restore log ($new_lines new lines) ---"
        # FAST: use awk to prefix lines in a single call (no bash while-read loop)
        # 用 || true 保护管道，避免 set -o pipefail 触发脚本立即退出
        tail -n "$new_lines" "$RESTORE_LOG_FILE" | awk -v ts="[$(date -u +%Y-%m-%dT%H:%M:%SZ)]" '{print ts, $0}' || true
        LAST_RESTORE_LINE="$current_lines"
        TIME_NO_NEW_LOG=0
    fi

    local current_size
    current_size=$(stat -c%s "$RESTORE_LOG_FILE" 2>/dev/null || echo "0")
    if [[ "$current_size" != "$LAST_LOG_SIZE" ]]; then
        LAST_LOG_SIZE="$current_size"
    fi
}

while true; do
    # 首次进入循环时显示诊断信息
    if [[ $WAITED -eq 0 ]]; then
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: waiting for restore completion..."
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint:   RESTORE_LOG_FILE=$RESTORE_LOG_FILE"
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint:   RESTORE_COMPLETED_FILE=$RESTORE_COMPLETED_FILE"
    fi

    # 每2秒：检查是否完成 + 获取最新日志
    show_restore_progress

    # 超时兜底：超过3600秒仍未完成，强制继续启动流程
    if [[ $WAITED -ge $MAX_WAIT_TIMEOUT ]]; then
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ⚠ WARNING: Restore timed out after ${WAITED}s ($((WAITED / 60))min), forcing proceed"
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ⚠ The restore process may still be running in the background"
        break
    fi

    # 检查恢复失败标志（发现失败立即退出，不等超时）
    if [[ -f "$RESTORE_FAILED_FILE" ]]; then
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ✗ Restore failed (flag detected), exiting to trigger HF Space restart"
        # 转储完整的恢复日志（不过滤关键字），确保用户看到失败前的所有信息
        _dump_restore_logs "restore failed flag detected"
        # ============================================================
        # FLUSHING OUTPUT — DO NOT EXIT PREMATURELY
        # Giving docker runtime 30+ seconds to drain stdout pipe.
        # Progressive FLUSH_MARKER_01..30 are emitted 1s apart —
        # the highest-numbered marker you see in docker logs tells
        # you how much output was captured before the container died.
        # ============================================================
        sync 2>/dev/null || true
        
        echo "========================================================================"
        echo "  ABOUT TO EXIT (code 1) — flushing output to docker logs..."
        echo "  Started: $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
        echo "  Diagnostics above: FULL dump of restore.log + gateway stderr"
        echo "========================================================================"
        sleep 1
        
        for _flush_n in $(seq 1 30); do
            printf -- "-- FLUSH_MARKER_%02d %s --\n" "$_flush_n" "$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
            sleep 1
        done
        
        echo "========================================================================"
        echo "  ALL DIAGNOSTICS FLUSHED TO STDOUT — exiting now"
        echo "  If you see this line and FLUSH_MARKER_30, output was FULLY captured"
        echo "========================================================================"
        sync 2>/dev/null || true
        sleep 1
        exit 1
    fi

    # 检查 openclaw-gateway 进程是否已异常退出（restore 过程中意外死亡）
    # 使用 supervisorctl status 作为权威检查，同时检查实际进程名作为兜底
    # 注意: supervisord 中命令是 openclaw-entrypoint.sh gateway，不包含字符串 "openclaw-gateway"
    #       因此 pgrep -f "openclaw-gateway" 永远不匹配，必须检查实际的进程名
    # 关键: 所有检查命令必须用 || true 保护，避免 set -e 误触发脚本立即退出
    gateway_alive=false
    gateway_fatal=false
    if [[ -f "/var/run/supervisord.pid" ]]; then
        # 方法1: 用 supervisorctl 检查 program 状态（最权威，匹配 program:openclaw-gateway）
        # supervisorctl status 在 program 不存在或状态不是 RUNNING 时返回非0，必须用 || true
        if command -v supervisorctl >/dev/null 2>&1; then
            svc_status=$(supervisorctl status openclaw-gateway 2>/dev/null || true)
            if echo "$svc_status" | grep -qE "(RUNNING|STARTING)"; then
                gateway_alive=true
            elif echo "$svc_status" | grep -qE "(FATAL|BACKOFF)"; then
                gateway_fatal=true
            fi
        fi
        # 方法2: 兜底检查 - 检查实际运行的入口脚本或恢复进程
        # pgrep 在无匹配时返回 1，在 if 条件中不会触发 set -e，但为了安全用 || true
        if [[ "$gateway_alive" != true ]]; then
            if { pgrep -f "openclaw-entrypoint" >/dev/null 2>&1 || true; } \
               || { pgrep -f "backup\.py restore" >/dev/null 2>&1 || true; } \
               || { pgrep -f "openclaw-restore" >/dev/null 2>&1 || true; }; then
                gateway_alive=true
            fi
        fi
    fi
    if [[ "$gateway_alive" == true ]]; then
        gateway_was_seen_alive=true
    fi
    if [[ "$gateway_alive" != true ]]; then
        # 如果 gateway 进程已经不存在，且未看到完成标志，视为恢复失败
        # 规则: 1) 如果 supervisord 报告 FATAL/BACKOFF → 立即检测为崩溃（无可救药）
        #       2) 如果曾经看到 gateway alive（gateway_was_seen_alive=true），立即检测为崩溃
        #       3) 如果从未看到 alive，等待 10 秒后再判定（确保 supervisord 有时间启动它）
        if [[ ! -f "$RESTORE_COMPLETED_FILE" ]] && { [[ "$gateway_fatal" == true ]] || [[ "$gateway_was_seen_alive" == true ]] || [[ $WAITED -gt 10 ]]; }; then
            if [[ "$gateway_fatal" == true ]]; then fatal_state="FATAL"; else fatal_state="DEAD"; fi
            echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] hf-entrypoint: ✗ openclaw-gateway process exited before restore completed (supervisord_state=${fatal_state}, waited=${WAITED}s), marking as failed"
            # 在退出前转储完整的恢复日志，让用户看到崩溃前发生了什么
            _dump_restore_logs "gateway process exited during restore (waited ${WAITED}s)"
            # ============================================================
            # FLUSHING OUTPUT — DO NOT EXIT PREMATURELY
            # Giving docker runtime 30+ seconds to drain stdout pipe.
            # Progressive FLUSH_MARKER_01..30 are emitted 1s apart —
            # the highest-numbered marker you see in docker logs tells
            # you how much output was captured before the container died.
            # ============================================================
            sync 2>/dev/null || true
            
            echo "========================================================================"
            echo "  ABOUT TO EXIT (code 1) — flushing output to docker logs..."
            echo "  Started: $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
            echo "  Diagnostics above: FULL dump of restore.log + gateway stderr"
            echo "========================================================================"
            sleep 1
            
            for _flush_n in $(seq 1 30); do
                printf -- "-- FLUSH_MARKER_%02d %s --\n" "$_flush_n" "$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
                sleep 1
            done
            
            echo "========================================================================"
            echo "  ALL DIAGNOSTICS FLUSHED TO STDOUT — exiting now"
            echo "  If you see this line and FLUSH_MARKER_30, output was FULLY captured"
            echo "========================================================================"
            sync 2>/dev/null || true
            sleep 1
            exit 1
        fi
    fi

    # 只检查恢复完成标志文件
    if [[ -f "$RESTORE_COMPLETED_FILE" ]]; then
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ✓ Restore completed"
        # 恢复成功后立即清除 fail-count，避免后续重启时被错误计数
        if [[ -f "$RESTORE_FAIL_COUNT_FILE" ]]; then
            rm -f "$RESTORE_FAIL_COUNT_FILE"
            echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: Cleared consecutive restore failure counter"
        fi
        # 恢复成功时也提供完整日志摘要（不过滤），确保隐藏的警告/错误也被看到
        _dump_log_file "RESTORE LOG (last 20 lines — restore completed successfully)" "$RESTORE_LOG_FILE" 20
        # 也检查 gateway 的 stderr，确认没有隐藏的 Python 错误
        if [[ -f "$OPENCLAW_STDERR_LOG" ]]; then
            _dump_log_file "GATEWAY STDERR (hidden Python errors/warnings check)" "$OPENCLAW_STDERR_LOG" 100
        fi
        # 清理 tail 转发进程（避免继续写入 stdout 干扰后续日志）
        if [[ -n "${TAIL_STDOUT_PID:-}" ]]; then
            kill "$TAIL_STDOUT_PID" 2>/dev/null || true
        fi
        if [[ -n "${TAIL_STDERR_PID:-}" ]]; then
            kill "$TAIL_STDERR_PID" 2>/dev/null || true
        fi
        break
    fi

    sleep 2
    WAITED=$((WAITED + 2))

    # 10-second early heartbeat so users do not see "stuck" behavior
    if [[ $WAITED -eq 10 ]]; then
        echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] hf-entrypoint: still waiting for restore... (${WAITED}s)"
        show_restore_progress
    fi

    # 如果日志无新内容，累计无更新时长
    if [[ -f "$RESTORE_LOG_FILE" ]]; then
        current_size=$(stat -c%s "$RESTORE_LOG_FILE" 2>/dev/null || echo "0")
        if [[ "$current_size" -eq "$LAST_LOG_SIZE" ]]; then
            TIME_NO_NEW_LOG=$((TIME_NO_NEW_LOG + 2))
        else
            TIME_NO_NEW_LOG=0
        fi
    fi

    # 每20秒输出一次聚合状态
    if [[ $((WAITED % PROGRESS_CHECK_INTERVAL)) -eq 0 ]]; then
        elapsed_min=$((WAITED / 60))
        log_size_str=""
        if [[ -f "$RESTORE_LOG_FILE" ]]; then
            file_size=$(stat -c%s "$RESTORE_LOG_FILE" 2>/dev/null || echo "0")
            if [[ $file_size -ge 1048576 ]]; then
                log_size_str="$((file_size / 1048576)).$(( (file_size % 1048576) * 10 / 1048576 ))MB"
            elif [[ $file_size -ge 1024 ]]; then
                log_size_str="$((file_size / 1024)).$(( (file_size % 1024) * 10 / 1024 ))KB"
            else
                log_size_str="${file_size}B"
            fi
        fi

        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: still waiting for restore... (${WAITED}s / ${elapsed_min}min, log: ${log_size_str:-N/A})"

        # 如果超过15分钟还没完成，发出预警

        # Also check gateway stderr — Python tracebacks live here
        if [[ -f "$OPENCLAW_STDERR_LOG" ]]; then
            gw_stderr_size=$(stat -c%s "$OPENCLAW_STDERR_LOG" 2>/dev/null || echo "0")
            if [[ "$gw_stderr_size" -gt "0" ]]; then
                echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] hf-entrypoint: gateway-stderr (last 30 lines, ${gw_stderr_size} bytes total) --"
                tail -n 30 "$OPENCLAW_STDERR_LOG" 2>/dev/null || true
                echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] hf-entrypoint: -- end gateway-stderr"
            fi
        fi

        if [[ $WAITED -ge $SLOW_WARN_THRESHOLD ]] && [[ $((WAITED % 60)) -eq 0 ]]; then
            echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ⚠ WARNING: Restore is taking longer than expected (>${elapsed_min}min). Large backup (>10GB) may require more time."
        fi

        show_restore_progress
    fi
done

echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ✓ Restore wait completed (${WAITED}s / $((WAITED / 60))min), proceeding with PM2 startup"

# ============================================
# 1.4 检查上次 prune 是否失败（daemon prune 的告警）
# ============================================
PRUNE_FAILED_FILE="/tmp/openclaw-prune-failed"
if [[ -f "$PRUNE_FAILED_FILE" ]]; then
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ⚠ WARNING: Previous backup prune operation failed"
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ⚠ Old backup snapshots may not have been cleaned up"
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ⚠ LFS storage quota may grow over time"
    rm -f "$PRUNE_FAILED_FILE"
fi

# 1.5 确保 cron daemon 运行
if ! pgrep -x cron >/dev/null 2>&1; then
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: starting cron daemon..."
    /usr/sbin/cron
fi

# ============================================
# 1.4 检查备份系统状态
# ============================================
BACKUP_LOCK_FILE="/tmp/openclaw-backup/backup-chain-broken.lock"
if [[ -f "$BACKUP_LOCK_FILE" ]]; then
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ⚠ WARNING: Backup system is LOCKED"
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ⚠ Backup chain is incomplete or broken"
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ⚠ New backups (full/incremental) are BLOCKED until fixed"
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ⚠ Lock reason:"
    sed 's/^/[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ⚠ /' "$BACKUP_LOCK_FILE"
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ⚠ To fix: ensure full backup exists in HuggingFace Dataset, then run:"
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ⚠   rm $BACKUP_LOCK_FILE"
else
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: ✓ Backup system is operational"
fi

# ============================================
# 2. 启动 PM2 管理, 附加的 node 进程（如果需要）
# ============================================
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: starting PM2 for others..."
mkdir -p /root/.pm2 /var/log/hf-entrypoint

if grep -qE '"name"\s*:' /app/pm2/ecosystem.config.js 2>/dev/null; then
    /usr/bin/pm2-runtime /app/pm2/ecosystem.config.js \
        >> /var/log/hf-entrypoint/pm2-stdout.log \
        2>> /var/log/hf-entrypoint/pm2-stderr.log &
    PM2_PID=$!
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: PM2 started (pid=$PM2_PID)"
else
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: PM2: no applications defined in ecosystem.config.js, skipping..."
    PM2_PID=""
fi

# ============================================
# 3. 信号转发（确保 PID 1 的 SIGTERM 能传到 supervisord）
# ============================================
signal_handler() {
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: received SIGTERM, forwarding to all processes..."
    # 停止SSH看门狗
    if [ -n "${SSH_WATCHDOG_PID:-}" ]; then
        echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: stopping SSH watchdog (pid=$SSH_WATCHDOG_PID)..."
        kill -TERM "$SSH_WATCHDOG_PID" 2>/dev/null || true
    fi
    # 停止supervisord
    kill -TERM "$SUPERVISORD_PID" 2>/dev/null || true
    # 停止PM2
    kill -TERM "$PM2_PID" 2>/dev/null || true
    # 停止SSH服务
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: stopping SSH service..."
    if pgrep -x "sshd" > /dev/null 2>&1; then
        killall sshd 2>/dev/null || true
    fi
    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: all services stopped"
}

trap signal_handler TERM INT QUIT

# ============================================
# 5. 启动 node hf-server.js 作为 PID 1
# ============================================
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] hf-entrypoint: starting node server.js..."
cd /app
exec node hf-server.js