entrypoint.sh

#!/bin/bash
# Entrypoint script for Hermes Agent on Hugging Face Spaces
# Based on Hermes Agent real config.yaml format (source: cli-config.yaml.example + hermes_cli/config.py)
#
# Startup architecture:
#   entrypoint.sh
#     ├── data_sync daemon      (background, data persistence)   ← FIXED: was missing
#     ├── Agent Bridge          (background, Unix socket IPC)
#     ├── hermes gateway run    (background, API Server :8642 + messaging platforms)
#     └── node hermes-web-ui    (foreground, BFF :7860)
#

set -e

echo "🚀 Hermes Agent v0.10.0 - Hugging Face Spaces"
echo "=============================================="

# Check required environment variables
if [ -z "$HF_DATASET_REPO" ]; then
    echo "⚠️  Warning: HF_DATASET_REPO is not set, data will not be persisted to Dataset"
fi

# ==================== INITIALIZE DIRECTORIES ====================
echo "📁 Initializing directory structure..."
mkdir -p /data/.hermes/{cron,sessions,logs,memories,skills,pairing,hooks,image_cache,audio_cache,whatsapp/session}
mkdir -p /data/.hermes-web-ui
mkdir -p /app/logs

# ==================== DATA RESTORE ====================
export SKIP_CONFIG_RESTORE=true

if [ -n "$HF_DATASET_REPO" ]; then
    echo "📥 Restoring data from Dataset..."
    python -m src.data_sync restore || {
        echo "⚠️  Data restore failed, starting with empty config"
    }
fi

# ==================== START DATA SYNC DAEMON ====================
# FIX: This block was missing from the original entrypoint — auto-backup never ran without it
if [ -n "$HF_DATASET_REPO" ]; then
    echo "💾 Starting data sync daemon..."
    python -m src.data_sync daemon &
    SYNC_PID=$!
    echo "   ✅ Sync daemon started (PID: $SYNC_PID, interval: ${SYNC_INTERVAL:-300}s)"
else
    echo "   ⚠️ HF_DATASET_REPO not set, skipping data sync"
    SYNC_PID=""
fi

# ==================== SYNC HISTORICAL SESSION MODEL CONFIG ====================
echo "🔄 Syncing historical session model config..."
python3 << 'SESSION_SYNC'
import json, os, glob
from pathlib import Path

current_model = os.environ.get('HERMES_MODEL', os.environ.get('MODEL_NAME', ''))
if not current_model:
    print("   ⚠️ No model configured, skipping session sync")
    exit(0)

sessions_dir = Path('/data/.hermes/sessions')
if not sessions_dir.exists():
    print("   ⚠️ Sessions directory does not exist")
    exit(0)

updated = 0
for session_file in sessions_dir.glob('*.json'):
    try:
        with open(session_file, 'r', encoding='utf-8') as f:
            data = json.load(f)
        
        if 'original_model' not in data and 'model' in data:
            data['original_model'] = data['model']
        
        if data.get('model') != current_model:
            data['model'] = current_model
            with open(session_file, 'w', encoding='utf-8') as f:
                json.dump(data, f, ensure_ascii=False, indent=2)
            updated += 1
    except Exception as e:
        print(f"   ⚠️ Failed to process {session_file.name}: {e}")

print(f"   ✅ Synced model config for {updated} sessions")
SESSION_SYNC

# ==================== MODEL CONFIGURATION SYSTEM ====================
echo "🤖 Configuring model system..."

# ---- Provider definitions ----
declare -A PROVIDER_MODELS=(
    ["xai"]="grok-4.3"
    ["nvidia"]="moonshotai/kimi-k2-thinking"
    ["siliconflow"]="deepseek-ai/DeepSeek-V4-Flash"
    ["openai"]="gpt-4o"
    ["anthropic"]="claude-3-5-sonnet-20241022"
    ["google"]="gemini-2.0-flash"
    ["gemini"]="gemini-2.5-flash"
    ["openrouter"]="meta-llama/llama-3.1-8b-instruct:free"
    ["longcat"]="LongCat-Flash-Thinking-2601"
)

declare -A PROVIDER_API_KEYS=(
    ["xai"]="XAI_API_KEY"
    ["nvidia"]="NVIDIA_API_KEY"
    ["siliconflow"]="SILICONFLOW_API_KEY"
    ["openai"]="OPENAI_API_KEY"
    ["anthropic"]="ANTHROPIC_API_KEY"
    ["google"]="GOOGLE_API_KEY"
    ["gemini"]="GEMINI_API_KEY"
    ["openrouter"]="OPENROUTER_API_KEY"
    ["longcat"]="LONGCAT_API_KEY"
)

declare -A PROVIDER_BASE_URLS=(
    ["xai"]="https://api.x.ai/v1"
    ["nvidia"]="https://integrate.api.nvidia.com/v1"
    ["siliconflow"]="https://api.siliconflow.cn/v1"
    ["openai"]="https://api.openai.com/v1"
    ["anthropic"]="https://api.anthropic.com/v1"
    ["google"]="https://generativelanguage.googleapis.com"
    ["gemini"]="https://generativelanguage.googleapis.com"
    ["openrouter"]="https://openrouter.ai/api/v1"
    ["longcat"]="https://api.longcat.chat/openai"
)

# ---- Detect main model ----
detect_main_model() {
    if [ -n "$MODEL_PROVIDER" ] && [ -n "$MODEL_NAME" ]; then
        echo "manual:$MODEL_PROVIDER:$MODEL_NAME"
        return
    fi
    for provider in xai nvidia siliconflow openai anthropic google openrouter longcat; do
        api_key_var="${PROVIDER_API_KEYS[$provider]}"
        if [ -n "${!api_key_var}" ]; then
            if [ -n "$MODEL_NAME" ]; then
                echo "auto:$provider:$MODEL_NAME"
            else
                echo "auto:$provider:${PROVIDER_MODELS[$provider]}"
            fi
            return
        fi
    done
    if [ -n "$GEMINI_API_KEY" ]; then
        echo "auto:gemini:${PROVIDER_MODELS[gemini]}"
        return
    fi
    echo "default:nvidia:${PROVIDER_MODELS[nvidia]}"
}

# Map provider name to what Hermes Gateway actually expects
map_provider_for_gateway() {
    local provider="$1"
    case "$provider" in
        siliconflow)
            echo "custom"
            ;;
        *)
            echo "$provider"
            ;;
    esac
}

# ---- Detect auxiliary models ----
detect_vision_model() {
    if [ -n "$VISION_MODEL" ]; then echo "$VISION_MODEL"; return; fi
    if [ -n "$NVIDIA_API_KEY" ]; then echo "meta/llama-3.2-11b-vision-instruct"; return; fi
    echo ""
}

detect_aux_model() {
    if [ -n "$AUX_MODEL" ]; then echo "$AUX_MODEL"; return; fi
    if [ -n "$OPENROUTER_API_KEY" ]; then echo "google/gemini-3-flash-preview"; return; fi
    if [ -n "$NVIDIA_API_KEY" ]; then echo "openai/gpt-oss-120b"; return; fi
    echo ""
}

detect_delegation_model() {
    if [ -n "$DELEGATION_MODEL" ]; then echo "$DELEGATION_MODEL"; return; fi
    if [ -n "$NVIDIA_API_KEY" ]; then echo "moonshotai/kimi-k2.6"; return; fi
    echo ""
}

# ---- Run detection ----
echo ""
echo "📋 Model configuration detection:"
echo "────────────────────────────────────────"

MAIN_DETECTED=$(detect_main_model)
IFS=':' read -r MAIN_MODE MAIN_PROVIDER MAIN_MODEL <<< "$MAIN_DETECTED"
echo "🎯 Main Model: $MAIN_PROVIDER/$MAIN_MODEL (mode: $MAIN_MODE)"

VISION_MODEL_VAL=$(detect_vision_model)
echo "👁️  Vision Model: ${VISION_MODEL_VAL:-auto-detect}"

AUX_MODEL_VAL=$(detect_aux_model)
echo "⚡ Aux Model: ${AUX_MODEL_VAL:-auto-detect}"

DELEGATION_MODEL_VAL=$(detect_delegation_model)
echo "💻 Delegation Model: ${DELEGATION_MODEL_VAL:-inherit-main}"

MAIN_BASE_URL="${PROVIDER_BASE_URLS[$MAIN_PROVIDER]}"
echo "   Base URL: $MAIN_BASE_URL"

echo "────────────────────────────────────────"

# ==================== GENERATE config.yaml ====================
CONFIG_FILE="/data/.hermes/config.yaml"
echo "📝 Generating config.yaml (Hermes real format)..."

infer_provider() {
    local model_id="$1"
    if [[ "$model_id" == google/* ]]; then echo "google"
    elif [[ "$model_id" == openrouter/* ]]; then echo "openrouter"
    elif [[ "$model_id" == Pro/* ]]; then echo "custom"
    else echo "$MAIN_PROVIDER"; fi
}

GATEWAY_MAIN_PROVIDER=$(map_provider_for_gateway "$MAIN_PROVIDER")
VISION_PROVIDER_VAL=$(map_provider_for_gateway "$(infer_provider "$VISION_MODEL_VAL")")
AUX_PROVIDER_VAL=$(map_provider_for_gateway "$(infer_provider "$AUX_MODEL_VAL")")
DELEGATION_PROVIDER_VAL=$(map_provider_for_gateway "$(infer_provider "$DELEGATION_MODEL_VAL")")

echo "🔄 Gateway provider mapping: $MAIN_PROVIDER → $GATEWAY_MAIN_PROVIDER"

cat > "$CONFIG_FILE" << EOF
# Hermes Agent Configuration
# Generated by entrypoint.sh at $(date -Iseconds)

# Main model configuration
model:
  default: "$MAIN_MODEL"
  provider: "$GATEWAY_MAIN_PROVIDER"
  base_url: "$MAIN_BASE_URL"

# Auxiliary model configuration (per-task overrides)
auxiliary:
  vision:
    provider: "${VISION_PROVIDER_VAL:-auto}"
    model: "${VISION_MODEL_VAL}"
    timeout: 120
    download_timeout: 30
  web_extract:
    provider: "${AUX_PROVIDER_VAL:-auto}"
    model: "${AUX_MODEL_VAL}"
    timeout: 360
  compression:
    provider: "${AUX_PROVIDER_VAL:-auto}"
    model: "${AUX_MODEL_VAL}"
    timeout: 120
  title_generation:
    provider: "${AUX_PROVIDER_VAL:-auto}"
    model: "${AUX_MODEL_VAL}"
    timeout: 30
  session_search:
    provider: "auto"
    model: ""
    timeout: 30
  skills_hub:
    provider: "auto"
    model: ""
    timeout: 30
  approval:
    provider: "auto"
    model: ""
    timeout: 30
  mcp:
    provider: "auto"
    model: ""
    timeout: 30
  flush_memories:
    provider: "auto"
    model: ""
    timeout: 30

# Sub-agent (Delegation) configuration
delegation:
  model: "${DELEGATION_MODEL_VAL}"
  provider: "${DELEGATION_PROVIDER_VAL}"
  max_iterations: 50
  reasoning_effort: "medium"

# API Server configuration
api_server:
  enabled: true
  port: 8642
  host: "127.0.0.1"

# Terminal configuration
terminal:
  backend: local
  timeout: 300
  shell: /bin/bash

# Display configuration
display:
  skin: default
  show_tool_progress: true
  show_resume: true
  spinner: dots

# Agent configuration
agent:
  max_iterations: 50
  approval_mode: ask
  dangerous_command_approval: ask
  gateway_timeout: 300

# Memory configuration
memory:
  enabled: true
  provider: local

# Compression configuration
compression:
  enabled: true
  threshold: 0.50

# Scheduled tasks
cron:
  enabled: true
  tick_interval: 60
EOF

echo "   ✅ Config file generated"

# ==================== MERGE USER CONFIG ====================
RESTORED_CONFIG="/data/.hermes/config.yaml.restored"
if [ -f "$RESTORED_CONFIG" ]; then
    echo "🔄 Merging user config (platforms, display, agent, etc.)..."
    python3 << 'MERGE_SCRIPT'
import yaml
import sys

GENERATED = '/data/.hermes/config.yaml'
RESTORED = '/data/.hermes/config.yaml.restored'

ENTRYPOINT_PRIORITY = {'model', 'auxiliary', 'delegation', 'api_server'}
USER_PRIORITY = {'platforms', 'display', 'agent', 'memory', 'compression', 'cron', 'terminal'}

try:
    with open(GENERATED) as f:
        generated = yaml.safe_load(f) or {}
    with open(RESTORED) as f:
        restored = yaml.safe_load(f) or {}

    merged = {}

    all_keys = set(list(generated.keys()) + list(restored.keys()))

    for key in all_keys:
        if key in ENTRYPOINT_PRIORITY:
            if key in generated:
                merged[key] = generated[key]
        elif key in USER_PRIORITY:
            if key in restored:
                merged[key] = restored[key]
            elif key in generated:
                merged[key] = generated[key]
        else:
            if key in restored:
                merged[key] = restored[key]
            elif key in generated:
                merged[key] = generated[key]

    with open(GENERATED, 'w') as f:
        yaml.dump(merged, f, default_flow_style=False, allow_unicode=True, sort_keys=False)

    merged_user_keys = [k for k in USER_PRIORITY if k in restored]
    print(f"   ✅ Merged user sections: {', '.join(merged_user_keys) if merged_user_keys else 'none'}")

except Exception as e:
    print(f"   ⚠️ Config merge failed: {e}, using generated default config")
    sys.exit(0)
MERGE_SCRIPT
    rm -f "$RESTORED_CONFIG"
else
    echo "   ℹ️ No merge needed (no restored user config)"
fi

# ==================== EXPORT PROVIDER ENVIRONMENT VARIABLES ====================
echo "🌐 Setting provider environment variables..."

for var in XAI_API_KEY NVIDIA_API_KEY SILICONFLOW_API_KEY OPENAI_API_KEY ANTHROPIC_API_KEY GOOGLE_API_KEY GEMINI_API_KEY OPENROUTER_API_KEY LONGCAT_API_KEY; do
    if [ -n "${!var}" ]; then
        export "$var"
    fi
done

if [ -n "$XAI_API_KEY" ]; then
    export XAI_BASE_URL="${XAI_BASE_URL:-https://api.x.ai/v1}"
fi
if [ -n "$NVIDIA_API_KEY" ]; then
    export NVIDIA_BASE_URL="${NVIDIA_BASE_URL:-https://integrate.api.nvidia.com/v1}"
fi
if [ -n "$SILICONFLOW_API_KEY" ]; then
    export SILICONFLOW_BASE_URL="${SILICONFLOW_BASE_URL:-https://api.siliconflow.cn/v1}"
    if [ -z "$OPENAI_API_KEY" ]; then
        export OPENAI_API_KEY="$SILICONFLOW_API_KEY"
        export OPENAI_BASE_URL="${SILICONFLOW_BASE_URL:-https://api.siliconflow.cn/v1}"
        echo "   ℹ️  Copied SILICONFLOW_API_KEY to OPENAI_API_KEY (Gateway compatibility)"
    fi
fi
if [ -n "$GEMINI_API_KEY" ]; then
    export GEMINI_BASE_URL="${GEMINI_BASE_URL:-https://generativelanguage.googleapis.com}"
fi
if [ -n "$OPENROUTER_API_KEY" ]; then
    export OPENROUTER_BASE_URL="${OPENROUTER_BASE_URL:-https://openrouter.ai/api/v1}"
fi
if [ -n "$LONGCAT_API_KEY" ]; then
    export LONGCAT_BASE_URL="${LONGCAT_BASE_URL:-https://api.longcat.chat/openai}"
fi

# Export API Server environment variables
export API_SERVER_ENABLED=true
export API_SERVER_PORT=8642
export API_SERVER_HOST=127.0.0.1

# Generate or reuse API Server key
if [ -n "$API_SERVER_KEY" ]; then
    echo "   ℹ️  Using existing API_SERVER_KEY"
else
    API_SERVER_KEY=$(python3 -c "import secrets; print(secrets.token_hex(32))")
    echo "   🔑 Generated new API_SERVER_KEY"
fi
export API_SERVER_KEY

# Allow all users by default
export GATEWAY_ALLOW_ALL_USERS="${GATEWAY_ALLOW_ALL_USERS:-true}"

# Export HERMES_HOME
export HERMES_HOME=/data/.hermes

# Export HERMES_MODEL
export HERMES_MODEL="$MAIN_MODEL"

echo "   ✅ API Key environment variables exported"
echo "   ✅ Base URL environment variables set"
echo "   ✅ API Server environment variables set (port: 8642)"
echo "   ✅ HERMES_HOME=$HERMES_HOME"
echo "   ✅ HERMES_MODEL=$HERMES_MODEL (process-level model override)"

# ==================== INJECT ENVIRONMENT VARIABLES ====================
echo "⚙️  Injecting environment variables into .env..."
ENV_FILE="/data/.hermes/.env"
mkdir -p /data/.hermes

PERSISTENT_VARS=(
    "MODEL_PROVIDER" "MODEL_NAME" "HERMES_MODEL"
    "VISION_MODEL" "AUX_MODEL" "DELEGATION_MODEL"
    "XAI_API_KEY" "XAI_BASE_URL"
    "NVIDIA_API_KEY" "NVIDIA_BASE_URL"
    "SILICONFLOW_API_KEY" "SILICONFLOW_BASE_URL"
    "OPENAI_API_KEY"
    "ANTHROPIC_API_KEY"
    "GOOGLE_API_KEY" "GEMINI_API_KEY" "GEMINI_BASE_URL"
    "OPENROUTER_API_KEY" "OPENROUTER_BASE_URL"
    "LONGCAT_API_KEY" "LONGCAT_BASE_URL"
    "API_SERVER_ENABLED" "API_SERVER_PORT" "API_SERVER_HOST"
    "TELEGRAM_BOT_TOKEN" "TELEGRAM_ALLOWED_USERS" "TELEGRAM_PROXY"
    "DISCORD_BOT_TOKEN" "DISCORD_CLIENT_ID"
    "SLACK_BOT_TOKEN" "SLACK_APP_TOKEN" "SLACK_SIGNING_SECRET"
    "WHATSAPP_BUSINESS_ID" "WHATSAPP_PHONE_NUMBER" "WHATSAPP_ACCESS_TOKEN"
    "WEIXIN_ACCOUNT_ID" "WEIXIN_TOKEN" "WEIXIN_BASE_URL"
    "GATEWAY_ALLOW_ALL_USERS"
    "AUTH_TOKEN"
    "HF_TOKEN"
    "HF_DATASET_REPO"
    "API_SERVER_ENABLED" "API_SERVER_PORT" "API_SERVER_HOST" "API_SERVER_KEY"
)

declare -A env_entries=()
if [ -f "$ENV_FILE" ]; then
    while IFS= read -r line; do
        [[ "$line" =~ ^[[:space:]]*# ]] && continue
        [[ -z "${line// }" ]] && continue
        eq_idx="${line%%=*}"
        if [ -n "$eq_idx" ] && [ "$eq_idx" != "$line" ]; then
            env_entries["$eq_idx"]="$line"
        fi
    done < "$ENV_FILE"
fi

for var in "${PERSISTENT_VARS[@]}"; do
    if [ -n "${!var}" ]; then
        env_entries["$var"]="${var}=${!var}"
    fi
done

# Mask ALL API keys before writing to .env (security fix)
{
    for key in "${!env_entries[@]}"; do
        echo "${env_entries[$key]}"
    done
} | sort | python3 -c "
import sys, re
for line in sys.stdin:
    line = line.rstrip()
    # Mask HF tokens
    line = re.sub(r'(HF_TOKEN|HUGGING_FACE_HUB_TOKEN)=hf_[a-zA-Z0-9]+', r'\1=***masked***', line)
    line = re.sub(r'(HF_TOKEN|HUGGING_FACE_HUB_TOKEN)=.+', r'\1=***masked***', line)
    # Mask all other API keys and secrets
    line = re.sub(r'([A-Z_]*(API_KEY|SECRET|TOKEN)[A-Z_]*)=\S+', r'\1=***masked***', line)
    print(line)
" > "${ENV_FILE}.masked"
# Write unmasked version to actual .env (for runtime use), masked version for backup
{
    for key in "${!env_entries[@]}"; do
        echo "${env_entries[$key]}"
    done
} | sort > "$ENV_FILE"

RESTORED_COUNT=$(grep -c '=' "$ENV_FILE")
echo "   ✅ Written ${RESTORED_COUNT} environment variables (including restored persistent vars)"

# ==================== START AGENT BRIDGE (must be before Gateway) ====================
echo "🔌 Starting Agent Bridge..."
BRIDGE_ENDPOINT="ipc:///tmp/hermes-agent-bridge.sock"

# Auto-detect correct agent-root (prefer site-packages)
BRIDGE_AGENT_ROOT=""
for candidate in "/usr/local/lib/python3.11/site-packages/" "$HOME/.hermes/hermes-agent" "/opt/hermes/hermes-agent"; do
  if [ -f "$candidate/run_agent.py" ] || [ -d "$candidate/hermes_cli" ]; then
    BRIDGE_AGENT_ROOT="$candidate"
    break
  fi
done
if [ -z "$BRIDGE_AGENT_ROOT" ]; then
  BRIDGE_AGENT_ROOT="/usr/local/lib/python3.11/site-packages/"
fi

BRIDGE_HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"

# Ensure bridge script exists
BRIDGE_SCRIPT="/opt/hermes-web-ui/dist/server/agent-bridge/hermes_bridge.py"

if [ -f "$BRIDGE_SCRIPT" ]; then
  # Clean up old socket file
  rm -f /tmp/hermes-agent-bridge.sock
  
  # Start bridge service (background)
  python3 "$BRIDGE_SCRIPT" \
  --endpoint "$BRIDGE_ENDPOINT" \
  --agent-root "$BRIDGE_AGENT_ROOT" \
  --hermes-home "$BRIDGE_HERMES_HOME" \
  --worker-profile default \
  > /data/.hermes/logs/bridge.log 2>&1 &
  BRIDGE_PID=$!
  echo " ✅ Bridge service started (PID: $BRIDGE_PID, Agent-Root: $BRIDGE_AGENT_ROOT)"
  
  # Wait for bridge socket to be created (up to 10 seconds)
  echo " ⏳ Waiting for Bridge socket to be ready..."
  BRIDGE_READY=false
  for i in $(seq 1 10); do
    if [ -S "/tmp/hermes-agent-bridge.sock" ]; then
      BRIDGE_READY=true
      break
    fi
    # Check if process is still alive
    if ! kill -0 $BRIDGE_PID 2>/dev/null; then
      echo " ⚠️ Bridge process exited unexpectedly. Check logs: tail -f /data/.hermes/logs/bridge.log"
      break
    fi
    sleep 1
  done
  
  if [ "$BRIDGE_READY" = true ]; then
    echo " ✅ Bridge socket ready: /tmp/hermes-agent-bridge.sock"
  else
    echo " ⚠️ Bridge socket not ready, retrying with fallback agent-root..."
    # Try fallback paths
    for alt_root in "/usr/local/lib/python3.11/site-packages/" "$HOME/.hermes/hermes-agent"; do
      if [ "$alt_root" != "$BRIDGE_AGENT_ROOT" ]; then
        rm -f /tmp/hermes-agent-bridge.sock
        python3 "$BRIDGE_SCRIPT" --endpoint "$BRIDGE_ENDPOINT" --agent-root "$alt_root" --hermes-home "$BRIDGE_HERMES_HOME" --worker-profile default > /data/.hermes/logs/bridge.log 2>&1 &
        BRIDGE_PID=$!
        echo " 🔄 Trying fallback path: $alt_root (PID: $BRIDGE_PID)"
        sleep 2
        if [ -S "/tmp/hermes-agent-bridge.sock" ]; then
          echo " ✅ Bridge socket ready at fallback path: $alt_root"
          break
        fi
      fi
    done
  fi
  
  if [ "$BRIDGE_READY" = true ]; then
    # Final check: verify process is still alive
    if ! kill -0 $BRIDGE_PID 2>/dev/null; then
      echo " ⚠️ Bridge process has exited but socket still exists (possible zombie process)"
    fi
  fi
else
  echo " ❌ Bridge script not found: $BRIDGE_SCRIPT"
  echo " ⚠️ Skipping Bridge startup, but some features may not be available"
fi

# ==================== START GATEWAY (API Server + Messaging Platforms) ====================
echo "📡 Starting Hermes Gateway + API Server..."

MAIN_API_KEY_VAR="${PROVIDER_API_KEYS[$MAIN_PROVIDER]}"
MAIN_API_KEY_VAL="${!MAIN_API_KEY_VAR}"
echo "🔍 Gateway configuration diagnostics:"
echo "   Original Provider: $MAIN_PROVIDER"
echo "   Gateway Provider:  $GATEWAY_MAIN_PROVIDER"
echo "   Model:             $MAIN_MODEL"
echo "   Base URL:          $MAIN_BASE_URL"
if [ -n "$MAIN_API_KEY_VAL" ]; then
    echo "   API Key ($MAIN_API_KEY_VAR): set (length: ${#MAIN_API_KEY_VAL})"
else
    echo "   ⚠️  API Key ($MAIN_API_KEY_VAR): not set! Provider authentication may fail"
fi

echo "📋 Available provider list:"
hermes model 2>/dev/null | head -30 || echo "   ⚠️ Unable to get provider list"

echo "🔧 Running configuration diagnostics:"
hermes doctor 2>/dev/null | grep -E "(provider|model|config|error|warning)" | head -20 || echo "   ⚠️ hermes doctor not available"

GATEWAY_PIDFILE="/data/.hermes/gateway.pid"

# Gateway wrapper: auto-restart + crash recovery
(
    while true; do
        hermes gateway run --replace 2>&1 | while IFS= read -r line; do
            echo "$line"
            case "$line" in
                *"Gateway failed to connect"*)
                    echo "   ⚠️ Gateway messaging platform connection failed, API Server still usable, retrying in 30s..."
                    ;;
            esac
        done
        EXIT_CODE=${PIPESTATUS[0]}
        if [ "$EXIT_CODE" -ne 0 ]; then
            echo "   ⚠️ Gateway process exited (code=$EXIT_CODE), restarting in 30s..."
            sleep 30
        else
            echo "   🛑 Gateway exited normally (may have been replaced by BFF restartGateway)"
            sleep 5
            if [ -f "$GATEWAY_PIDFILE" ]; then
                NEW_PID=$(python3 -c "import json; print(json.load(open('$GATEWAY_PIDFILE')).get('pid',0))" 2>/dev/null || echo 0)
                if [ "$NEW_PID" -gt 0 ] && kill -0 "$NEW_PID" 2>/dev/null; then
                    echo "   🔄 Detected new gateway process (PID: $NEW_PID), waiting for it to exit..."
                    while kill -0 "$NEW_PID" 2>/dev/null; do sleep 5; done
                    echo "   ⚠️ New gateway process has exited, restarting wrapper in 30s..."
                    sleep 30
                    continue
                fi
            fi
            echo "   🛑 No new gateway process found, not restarting"
            break
        fi
    done
) &
GATEWAY_PID=$!

# Wait for API Server to be ready
echo "   ⏳ Waiting for API Server to be ready (:8642)..."
API_READY=false
for i in $(seq 1 30); do
    if curl -sf http://127.0.0.1:8642/health > /dev/null 2>&1; then
        API_READY=true
        break
    fi
    sleep 1
done

if [ "$API_READY" = true ]; then
    echo "   ✅ API Server ready (http://127.0.0.1:8642)"
else
    echo "   ⚠️ API Server not ready within 30s, continuing to start Web UI (API Server may become available later)"
fi

if kill -0 $GATEWAY_PID 2>/dev/null; then
    echo "   ✅ Gateway process running (PID: $GATEWAY_PID)"
else
    echo "   ⚠️ Gateway process has exited, only Web UI will be available"
fi

echo ""
echo "💡 Tips:"
echo "   - Channels page: configure WeChat, Feishu, WeCom and other platforms"
echo "   - Models page:   manage model providers"
echo "   - Jobs page:     manage scheduled tasks"
echo ""

# ==================== AUTH TOKEN SETUP ====================
echo "🔑 Configuring Web UI authentication..."
if [ -z "$AUTH_TOKEN" ]; then
    AUTH_TOKEN_FILE="/data/.hermes-web-ui/.token"
    if [ -f "$AUTH_TOKEN_FILE" ]; then
        AUTH_TOKEN=$(cat "$AUTH_TOKEN_FILE")
        echo "   ✅ Restored Web UI auth token"
    else
        AUTH_TOKEN=$(openssl rand -hex 16 2>/dev/null || head -c 32 /dev/urandom | xxd -p | head -c 32)
        mkdir -p /data/.hermes-web-ui
        echo "$AUTH_TOKEN" > "$AUTH_TOKEN_FILE"
        echo ""
        echo "   ╔══════════════════════════════════════════════════╗"
        echo "   ║  🔑 Web UI Auth Token (save this!)               ║"
        echo "   ║  $AUTH_TOKEN"
        echo "   ║                                                    ║"
        echo "   ║  Enter this token on the Web UI login page        ║"
        echo "   ║  Or set AUTH_TOKEN in HF Spaces Settings          ║"
        echo "   ╚══════════════════════════════════════════════════╝"
        echo ""
    fi
else
    echo "   ✅ Using AUTH_TOKEN from environment variable"
fi
export AUTH_TOKEN

# ==================== FIX HERMES-WEB-UI AGENT BRIDGE ====================
echo "🔧 Fixing agent bridge permissions..."

BRIDGE_PATHS=(
    "/opt/hermes-web-ui/dist/server/agent-bridge/hermes_bridge.py"
    "/usr/lib/node_modules/hermes-web-ui/dist/server/agent-bridge/hermes_bridge.py"
)
BRIDGE_FOUND=false
for bridge_path in "${BRIDGE_PATHS[@]}"; do
    if [ -f "$bridge_path" ]; then
        chmod +x "$bridge_path"
        echo "   ✅ Bridge script permissions fixed: $bridge_path"
        BRIDGE_FOUND=true
    fi
done
if [ "$BRIDGE_FOUND" = false ]; then
    echo "   ⚠️ Bridge script not found in common paths, trying global search..."
    find /opt /usr -name "hermes_bridge.py" -type f 2>/dev/null | while read -r f; do
        chmod +x "$f"
        echo "   ✅ Permissions set: $f"
    done
fi

# Ensure hermes module is loadable by bridge
echo "🔍 Locating hermes package path..."

HERMES_PKG_PATH=""

if [ -z "$HERMES_PKG_PATH" ]; then
    for candidate in $(find /usr/local/lib/python*/site-packages -maxdepth 1 -type d \( -name "hermes" -o -name "hermes_agent" -o -name "hermes_cli" \) 2>/dev/null); do
        if [ -f "$candidate/__init__.py" ]; then
            HERMES_PKG_PATH="$candidate"
            echo "   ✅ Method 1 found hermes package: $HERMES_PKG_PATH"
            break
        fi
    done
fi

if [ -z "$HERMES_PKG_PATH" ]; then
    PIP_LOCATION=$(python3 -m pip show hermes-agent 2>/dev/null | grep ^Location | cut -d' ' -f2-)
    if [ -n "$PIP_LOCATION" ]; then
        for name in hermes hermes_agent hermes_cli; do
            if [ -d "$PIP_LOCATION/$name" ] && [ -f "$PIP_LOCATION/$name/__init__.py" ]; then
                HERMES_PKG_PATH="$PIP_LOCATION/$name"
                echo "   ✅ Method 2 found hermes package: $HERMES_PKG_PATH"
                break
            fi
        done
    fi
fi

if [ -z "$HERMES_PKG_PATH" ]; then
    HERMES_CMD=$(which hermes 2>/dev/null)
    if [ -n "$HERMES_CMD" ]; then
        HERMES_MODULE=$(head -50 "$HERMES_CMD" 2>/dev/null | grep -E "from|import" | grep -oE "hermes[a-z_]*" | head -1)
        if [ -n "$HERMES_MODULE" ]; then
            HERMES_TRY=$(python3 -c "import $HERMES_MODULE; print($HERMES_MODULE.__path__[0])" 2>/dev/null)
            if [ -n "$HERMES_TRY" ]; then
                HERMES_PKG_PATH="$HERMES_TRY"
                echo "   ✅ Method 3 found hermes package: $HERMES_PKG_PATH (module: $HERMES_MODULE)"
            fi
        fi
    fi
fi

if [ -z "$HERMES_PKG_PATH" ]; then
    HERMES_PKG_PATH=$(python3 -c "
import sys, os
for p in sys.path:
    for name in ['hermes', 'hermes_agent', 'hermes_cli']:
        candidate = os.path.join(p, name)
        if os.path.isdir(candidate) and os.path.exists(os.path.join(candidate, '__init__.py')):
            print(candidate)
            sys.exit(0)
print('')
" 2>/dev/null)
    if [ -n "$HERMES_PKG_PATH" ]; then
        echo "   ✅ Method 4 found hermes package: $HERMES_PKG_PATH"
    fi
fi

if [ -z "$HERMES_PKG_PATH" ] && [ -d "/usr/local/lib/hermes-agent" ]; then
    HERMES_PKG_PATH="/usr/local/lib/hermes-agent"
    echo "   ✅ Method 5 found hermes package: $HERMES_PKG_PATH"
fi

HERMES_REAL_HOME=$(realpath ~/.hermes 2>/dev/null || echo "$HOME/.hermes")
mkdir -p "$HERMES_REAL_HOME"

if [ -n "$HERMES_PKG_PATH" ] && [ -d "$HERMES_PKG_PATH" ]; then
    if [ ! -e "$HERMES_REAL_HOME/hermes-agent" ] && [ ! -L "$HERMES_REAL_HOME/hermes-agent" ]; then
        ln -s "$HERMES_PKG_PATH" "$HERMES_REAL_HOME/hermes-agent"
        echo "   ✅ hermes-agent module link created → $HERMES_PKG_PATH"
    fi
    if [ ! -e "$HERMES_REAL_HOME/hermes" ] && [ ! -L "$HERMES_REAL_HOME/hermes" ]; then
        ln -s "$HERMES_PKG_PATH" "$HERMES_REAL_HOME/hermes"
        echo "   ✅ hermes module link created → $HERMES_PKG_PATH"
    fi
    
    PYTHON_SITE_PACKAGES=$(python3 -c "import site; print(site.getsitepackages()[0])" 2>/dev/null)
    if [ -n "$PYTHON_SITE_PACKAGES" ]; then
        export PYTHONPATH="${PYTHONPATH:+$PYTHONPATH:}$PYTHON_SITE_PACKAGES"
        echo "   ✅ PYTHONPATH set: $PYTHONPATH"
    fi
    
    echo "🔍 Testing bridge script..."
    if [ -f "$BRIDGE_SCRIPT" ]; then
        python3 "$BRIDGE_SCRIPT" --help 2>&1 | head -5 || echo "   ⚠️ Bridge script test failed"
    fi
else
    echo "   ❌ All methods failed to locate hermes package path!"
    echo "   📋 Diagnostic info:"
    echo "      Python site-packages:"
    python3 -c "import site; [print('     ', p) for p in site.getsitepackages()]" 2>/dev/null || true
    echo "      sys.path:"
    python3 -c "import sys; [print('     ', p) for p in sys.path if __import__('os').path.isdir(p)]" 2>/dev/null || true
    echo "      pip list | grep hermes:"
    python3 -m pip list 2>/dev/null | grep -i hermes || echo "        (no results)"
    echo "      which hermes: $(which hermes 2>/dev/null || echo 'not found')"
fi

# ==================== START WEB UI (BFF Server) ====================
echo "🌐 Starting Hermes Web UI..."
echo "   BFF Server: http://0.0.0.0:7860"
echo "   Upstream:   http://127.0.0.1:8642"
echo ""

export PORT=7860
export UPSTREAM=http://127.0.0.1:8642
export HERMES_BIN=/usr/local/bin/hermes
export HERMES_HOME=/data/.hermes

# Graceful shutdown handler
cleanup() {
    echo ""
    echo "🛑 Running cleanup..."

    if [ -n "$HF_DATASET_REPO" ]; then
        echo "   💾 Running final data backup..."
        python -m src.data_sync backup --force 2>/dev/null || echo "   ⚠️ Backup failed"
    fi

    # Stop processes in order: BFF → Bridge → Gateway → Sync
    if [ -n "$BRIDGE_PID" ] && kill -0 $BRIDGE_PID 2>/dev/null; then
        echo "   🛑 Stopping Agent Bridge..."
        kill $BRIDGE_PID 2>/dev/null || true
        wait $BRIDGE_PID 2>/dev/null || true
        rm -f /tmp/hermes-agent-bridge.sock 2>/dev/null
    fi
    if [ -n "$BFF_PID" ] && kill -0 $BFF_PID 2>/dev/null; then
        echo "   🛑 Stopping Web UI..."
        kill $BFF_PID 2>/dev/null || true
        wait $BFF_PID 2>/dev/null || true
    fi
    if [ -n "$GATEWAY_PID" ] && kill -0 $GATEWAY_PID 2>/dev/null; then
        echo "   🛑 Stopping Gateway..."
        kill $GATEWAY_PID 2>/dev/null || true
        wait $GATEWAY_PID 2>/dev/null || true
    fi
    if [ -n "$SYNC_PID" ] && kill -0 $SYNC_PID 2>/dev/null; then
        echo "   🛑 Stopping data sync..."
        kill $SYNC_PID 2>/dev/null || true
        wait $SYNC_PID 2>/dev/null || true
    fi

    echo "👋 Goodbye!"
    exit 0
}

trap cleanup SIGTERM SIGINT

# Start BFF Server (bind directly to port 7860)
PORT=7860 node /opt/hermes-web-ui/dist/server/index.js &
BFF_PID=$!

# Wait for BFF to be ready
echo "   ⏳ Waiting for Web UI to be ready..."
BFF_READY=false
for i in $(seq 1 20); do
    if curl -sf http://localhost:7860/health > /dev/null 2>&1; then
        BFF_READY=true
        break
    fi
    sleep 1
done

if [ "$BFF_READY" = true ]; then
    echo "   ✅ Web UI ready → http://localhost:7860"
else
    echo "   ⚠️ Web UI not ready within 20s, check logs"
fi

# Re-verify model config after BFF startup
if [ -f "$CONFIG_FILE" ]; then
    if command -v yq &>/dev/null; then
        ACTUAL_MODEL=$(yq '.model.default' "$CONFIG_FILE" 2>/dev/null)
        if [ -n "$ACTUAL_MODEL" ] && [ "$ACTUAL_MODEL" != "$MAIN_MODEL" ] && [ "$ACTUAL_MODEL" != "null" ]; then
            echo "   ⚠️ Model was overwritten by BFF startup process!"
            echo "   📋 Expected: $MAIN_MODEL, Actual: $ACTUAL_MODEL"
            echo "   🔒 Re-writing correct model config..."
            yq -i ".model.default = \"$MAIN_MODEL\"" "$CONFIG_FILE"
            yq -i ".model.provider = \"$GATEWAY_MAIN_PROVIDER\"" "$CONFIG_FILE"
            yq -i ".model.base_url = \"$MAIN_BASE_URL\"" "$CONFIG_FILE"
            echo "   ✅ Model corrected: $GATEWAY_MAIN_PROVIDER/$MAIN_MODEL"
        elif [ -z "$ACTUAL_MODEL" ] || [ "$ACTUAL_MODEL" = "null" ]; then
            echo "   ⚠️ Model field is empty! Re-writing..."
            yq -i ".model.default = \"$MAIN_MODEL\"" "$CONFIG_FILE"
            yq -i ".model.provider = \"$GATEWAY_MAIN_PROVIDER\"" "$CONFIG_FILE"
            yq -i ".model.base_url = \"$MAIN_BASE_URL\"" "$CONFIG_FILE"
            echo "   ✅ Model corrected: $GATEWAY_MAIN_PROVIDER/$MAIN_MODEL"
        else
            echo "   ✅ Model config verified: $GATEWAY_MAIN_PROVIDER/$MAIN_MODEL"
        fi
    fi
fi

# Wait for BFF main process (foreground block — container lifetime controlled by BFF)
wait $BFF_PID