Spaces:

Jishnupg
/

Opencode-Cli

Running

Hermes Agent

fix(sqlite): eliminate concurrent lock collisions, remove startup backup interruption, and clean stale WAL frames

6fef1ae about 18 hours ago

20 kB

#!/usr/bin/env bash
set -Eeuo pipefail

START_TIME=$(date +%s)
FASTAPI_PID=""
NGINX_PID=""
OMNIROUTE_PID=""
OMNIROUTE_FAIL_COUNT=0
OMNIROUTE_BACKOFF_S=2
get_elapsed() {
    if [ -n "$START_TIME" ]; then
        python3 -c "import time; print(f'{time.time() - $START_TIME:.2f}s')" 2>/dev/null || echo "0.0s"
    else
        echo "0.0s"
    fi
}

log_info() {
    echo "$(date -u '+%Y-%m-%d %H:%M:%S') [INFO] [$1] $2"
}

log_warn() {
    echo "$(date -u '+%Y-%m-%d %H:%M:%S') [WARN] [$1] $2"
}

log_error() {
    echo "$(date -u '+%Y-%m-%d %H:%M:%S') [ERROR] [$1] $2"
}

log_info "SYSTEM" "============================================"
log_info "SYSTEM" "=== OpenCode Space Container Starting ==="
log_info "SYSTEM" "============================================"

# Step 1: Configure Git & Environment Secrets Validation
git config --global --add safe.directory '*' 2>/dev/null || true

# Step 2: Ensure persistent /data directory structure exists
log_info "INIT" "Setting up /data persistent volume directories..."
mkdir -p /data/share/opencode /data/config/opencode /data/cache/opencode /data/state/opencode 2>/dev/null || true
mkdir -p /root/.cache /data/cache 2>/dev/null || true

# Ensure persistent volume directories exist (Never wipe user data on restarts)
mkdir -p /data/omniroute/backups /data/omniroute/oauth /data/omniroute/credentials /data/omniroute/call_logs /data/omniroute/runtime 2>/dev/null || true
# Clean only legacy non-OmniRoute directories if lingering
rm -rf /data/jellyfin /data/hermes /root/.hermes /data/share 2>/dev/null || true
chmod -R 777 /root/.cache /data/cache /data/omniroute 2>/dev/null || true

# Playwright Browser Metadata Fix for gemini-web / browser providers
export PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
export PLAYWRIGHT_BROWSERS_PATH=/root/.cache/ms-playwright
mkdir -p /root/.cache/ms-playwright /omniroute/.cache/ms-playwright 2>/dev/null || true
python3 -c "
import os, json
browsers_json = {
  'browsers': [
    {'name': 'chromium', 'revision': '1112', 'installByDefault': True},
    {'name': 'firefox', 'revision': '1448', 'installByDefault': True},
    {'name': 'webkit', 'revision': '2007', 'installByDefault': True},
    {'name': 'ffmpeg', 'revision': '1009', 'installByDefault': True}
  ]
}
targets = [
    '/root/.cache/ms-playwright',
    '/omniroute/.cache/ms-playwright',
    '/omniroute/node_modules/playwright-core',
    '/omniroute/node_modules/playwright/node_modules/playwright-core',
    '/usr/local/lib/node_modules/playwright-core',
    '/usr/local/lib/node_modules/playwright/node_modules/playwright-core'
]
if os.path.exists('/omniroute'):
    for root, dirs, files in os.walk('/omniroute'):
        if os.path.basename(root) == 'playwright-core':
            targets.append(root)
for target in set(targets):
    try:
        os.makedirs(target, exist_ok=True)
        b_path = os.path.join(target, 'browsers.json')
        if not os.path.exists(b_path) or os.path.getsize(b_path) == 0:
            with open(b_path, 'w', encoding='utf-8') as f:
                json.dump(browsers_json, f, indent=2)
            print(f'[PLAYWRIGHT] Created browsers.json at {b_path}')
    except Exception:
        pass
" 2>/dev/null || true

# ── STEP 1: Master Secret Validation & Persistent Encryption Keys ─────────────
mkdir -p /data/omniroute 2>/dev/null || true

if [ -n "$STORAGE_ENCRYPTION_KEY" ]; then
    echo "$STORAGE_ENCRYPTION_KEY" > /data/omniroute/.storage_encryption_key 2>/dev/null || true
elif [ -f "/data/omniroute/.storage_encryption_key" ] && [ -s "/data/omniroute/.storage_encryption_key" ]; then
    export STORAGE_ENCRYPTION_KEY=$(cat /data/omniroute/.storage_encryption_key | tr -d '\r\n')
    echo "[PERSISTENCE] Loaded persistent STORAGE_ENCRYPTION_KEY from /data/omniroute/.storage_encryption_key"
else
    export STORAGE_ENCRYPTION_KEY=$(python3 -c "import secrets; print(secrets.token_hex(16))" 2>/dev/null || echo "omniroute_default_storage_key_32bytes")
    echo "$STORAGE_ENCRYPTION_KEY" > /data/omniroute/.storage_encryption_key 2>/dev/null || true
    echo "[PERSISTENCE] Generated and saved persistent STORAGE_ENCRYPTION_KEY to /data/omniroute"
fi

if [ -n "$JWT_SECRET" ]; then
    echo "$JWT_SECRET" > /data/omniroute/.jwt_secret 2>/dev/null || true
elif [ -f "/data/omniroute/.jwt_secret" ] && [ -s "/data/omniroute/.jwt_secret" ]; then
    export JWT_SECRET=$(cat /data/omniroute/.jwt_secret | tr -d '\r\n')
    echo "[PERSISTENCE] Loaded persistent JWT_SECRET from /data/omniroute/.jwt_secret"
else
    export JWT_SECRET=$(python3 -c "import secrets; print(secrets.token_hex(16))" 2>/dev/null || echo "omniroute_default_jwt_secret_key_32bytes")
    echo "$JWT_SECRET" > /data/omniroute/.jwt_secret 2>/dev/null || true
    echo "[PERSISTENCE] Generated and saved persistent JWT_SECRET to /data/omniroute"
fi

if [ -n "$API_KEY_SECRET" ]; then
    echo "$API_KEY_SECRET" > /data/omniroute/.api_key_secret 2>/dev/null || true
elif [ -f "/data/omniroute/.api_key_secret" ] && [ -s "/data/omniroute/.api_key_secret" ]; then
    export API_KEY_SECRET=$(cat /data/omniroute/.api_key_secret | tr -d '\r\n')
    echo "[PERSISTENCE] Loaded persistent API_KEY_SECRET from /data/omniroute/.api_key_secret"
else
    export API_KEY_SECRET=$(python3 -c "import secrets; print(secrets.token_hex(16))" 2>/dev/null || echo "omniroute_default_api_key_secret_32bytes")
    echo "$API_KEY_SECRET" > /data/omniroute/.api_key_secret 2>/dev/null || true
    echo "[PERSISTENCE] Generated and saved persistent API_KEY_SECRET to /data/omniroute"
fi

export INITIAL_PASSWORD="${INITIAL_PASSWORD:-Jishnu2005}"
echo "$INITIAL_PASSWORD" > /data/omniroute/.initial_password 2>/dev/null || true
echo "[PERSISTENCE] Configured OmniRoute administrator password to persistent storage."

export ENCRYPTION_SECRET="${STORAGE_ENCRYPTION_KEY}"
export OMNIROUTE_SECRET_KEY="${STORAGE_ENCRYPTION_KEY}"
export OMNIROUTE_STORAGE_KEY="${STORAGE_ENCRYPTION_KEY}"
export OMNIROUTE_STORAGE_SECRET="${STORAGE_ENCRYPTION_KEY}"
export OMNIROUTE_WS_BRIDGE_SECRET="${OMNIROUTE_WS_BRIDGE_SECRET:-$(echo "ws_bridge_${JWT_SECRET}" | sha256sum | cut -c1-48)}"

# Purge any legacy secret env files from storage bucket to maintain strict secret separation
rm -f /data/.env /data/secrets.env /data/secrets /data/config/.env /data/omniroute/.env /data/omniroute/secrets.env /data/omniroute/secrets /data/omniroute/.secrets /data/omniroute/server.env 2>/dev/null || true

echo "[BOOT] Secrets validated: $(get_elapsed)"

# ── STEP 2: Canonical Direct Persistent Storage & Preflight Gate ──────────────
export DATA_DIR="/data/omniroute"
export OMNIROUTE_DATA_DIR="/data/omniroute"
export OMNIROUTE_RUNTIME_DIR="/root/.omniroute"
export OMNIROUTE_RUNTIME_DB="/root/.omniroute/storage.sqlite"
mkdir -p "$OMNIROUTE_RUNTIME_DIR" "$DATA_DIR/.incoming"

preflight_persistence_gate() {
    log_info "PREFLIGHT" "Validating persistent storage invariants for OmniRoute..."
    if [ ! -d "/data" ]; then
        log_error "PREFLIGHT" "CRITICAL: /data directory does not exist! Persistent bucket not mounted."
        return 1
    fi
    if [ ! -w "/data" ]; then
        log_error "PREFLIGHT" "CRITICAL: /data is not writable!"
        return 1
    fi
    mkdir -p "$DATA_DIR/backups" "$DATA_DIR/oauth" "$DATA_DIR/credentials" "$DATA_DIR/call_logs" "$DATA_DIR/runtime" 2>/dev/null || true
    if [ ! -w "$DATA_DIR" ]; then
        log_error "PREFLIGHT" "CRITICAL: DATA_DIR ($DATA_DIR) is not writable!"
        return 1
    fi

    # Touch probe to verify read-write access
    _PROBE="$DATA_DIR/.probe_$$"
    if ! touch "$_PROBE" 2>/dev/null; then
        log_error "PREFLIGHT" "CRITICAL: Write probe failed in $DATA_DIR!"
        return 1
    fi
    rm -f "$_PROBE" 2>/dev/null || true

    # Cloud Vault Restore: Synchronize clean, vacuumed authoritative database
    log_info "PREFLIGHT" "Synchronizing 100% verified vacuumed SQLite database from cloud vault..."
    rm -f "$DATA_DIR/storage.sqlite"* "$DATA_DIR/backups/last-known-good.sqlite"* "$OMNIROUTE_RUNTIME_DIR/storage.sqlite"* 2>/dev/null || true
    if [ -f "/vault_sync.py" ]; then
        python3 /vault_sync.py restore --force || true
    fi

    log_info "PREFLIGHT" "Persistence preflight passed successfully. Single source of truth: $DATA_DIR"
    return 0
}

preflight_persistence_gate || log_error "PREFLIGHT" "Warning: persistence preflight reported issues, continuing with best effort."

# Runtime SQLite is authoritative; /data contains only verified snapshots.
if ! python3 /persistence_manager.py recover 2>&1 | tee /data/cache/persistence-recovery.log; then
    log_error "PERSISTENCE" "Verified runtime database recovery failed; refusing to start OmniRoute."
    exit 1
fi

# SQLite-Aware Backup Routine (Scheduled & On Shutdown)
backup_omniroute_db() {
    if [ -f "$OMNIROUTE_RUNTIME_DB" ]; then
        if python3 /persistence_manager.py snapshot >> /data/cache/persistence.log 2>&1; then
            log_info "BACKUP" "Verified SQLite snapshot promoted."
            if [ -f "/vault_sync.py" ]; then
                python3 /vault_sync.py backup >/dev/null 2>&1 &
            fi
        else
            log_warn "BACKUP" "Verified snapshot failed; previous persistent snapshot preserved."
        fi
    fi
}

shutdown_gracefully() {
    log_info "SHUTDOWN" "Process termination signal received. Flushing persistence state..."
    if [ -n "$OMNIROUTE_PID" ] && kill -0 $OMNIROUTE_PID 2>/dev/null; then
        kill -INT $OMNIROUTE_PID 2>/dev/null || true
        for _i in $(seq 1 10); do
            if ! kill -0 $OMNIROUTE_PID 2>/dev/null; then
                break
            fi
            sleep 1
        done
    fi
    if [ -f "$OMNIROUTE_RUNTIME_DB" ]; then
        backup_omniroute_db >/dev/null 2>&1 || true
        if [ -f "/vault_sync.py" ]; then
            python3 /vault_sync.py backup >/dev/null 2>&1 || true
        fi
    fi
    log_info "SHUTDOWN" "Persistence checkpoint complete. Container exiting."
    exit 0
}

trap shutdown_gracefully EXIT INT TERM

export HF_HUB_DISABLE_PROGRESS_BARS=1
export TQDM_DISABLE=1

# ── STEP 3: Start FastAPI Gateway Immediately ────────────────────────────────
echo "[BOOT] FastAPI starting: $(get_elapsed)"
cd /
python3 -m uvicorn proxy:app --host 127.0.0.1 --port 8000 --workers 2 --no-access-log --log-level warning &
FASTAPI_PID=$!

# Step 4: Wait for FastAPI /health/live to return HTTP 200
FASTAPI_READY=0
for i in $(seq 1 30); do
    if curl --max-time 2 -sf -i http://127.0.0.1:8000/health/live | grep -q "200 OK"; then
        FASTAPI_READY=1
        END_TIME=$(date +%s)
        ELAPSED=$((END_TIME - START_TIME))
        echo "[BOOT] FastAPI live: ${ELAPSED}s"
        break
    fi
    sleep 0.2
done

if [ "$FASTAPI_READY" -ne 1 ]; then
    echo "[WARN] FastAPI did not report ready within timeout, proceeding with startup..."
fi

# Step 5: Start Nginx Edge Server on Public Port 4096
echo "[BOOT] Nginx starting..."
rm -f /etc/nginx/sites-enabled/default /etc/nginx/sites-available/default 2>/dev/null || true
nginx -t -c /nginx.conf
nginx -g 'daemon off;' -c /nginx.conf &
NGINX_PID=$!

# Step 6: Wait for Nginx /health/live on Port 4096 to return HTTP 200
NGINX_READY=0
for i in $(seq 1 30); do
    if curl --max-time 2 -sf -i http://127.0.0.1:4096/health/live | grep -q "200 OK"; then
        NGINX_READY=1
        break
    fi
    sleep 0.2
done

# Step 7: Signal Public Gateway Readiness immediately to Hugging Face
echo "[BOOT] Public gateway live"
echo "[BOOT] Background services starting asynchronously..."

    # Step 8: Start Redis Server & OmniRoute AI Gateway in Background
    if command -v redis-server >/dev/null 2>&1; then
        echo "[INIT] Starting Redis server on port 6379..."
        redis-server --daemonize yes 2>/dev/null || true
        if command -v redis-cli >/dev/null 2>&1; then
            echo "[INIT] Redis server active on port 6379."
        fi
    fi

    echo "[INIT] Starting OmniRoute AI Gateway..."
    export PORT=20128
    export HOSTNAME="127.0.0.1"
    export DATA_DIR="$OMNIROUTE_RUNTIME_DIR"
    export OMNIROUTE_DATA_DIR="$OMNIROUTE_RUNTIME_DIR"
    export REDIS_URL="redis://127.0.0.1:6379"
    export NEXT_PUBLIC_BASE_URL="https://jishnupg-opencode-cli.hf.space"
    export AUTH_COOKIE_SECURE="true"
    export ALLOW_REMOTE_OAUTH="true"
    export ALLOW_BROWSER_OAUTH="true"
    export OAUTH_CALLBACK_URL="https://jishnupg-opencode-cli.hf.space/callback"
    export NEXT_PUBLIC_OAUTH_CALLBACK_URL="https://jishnupg-opencode-cli.hf.space/callback"
    export REDIRECT_URI="https://jishnupg-opencode-cli.hf.space/callback"
    export ANTIGRAVITY_OAUTH_REDIRECT_URI="https://jishnupg-opencode-cli.hf.space/callback"
    export ANTIGRAVITY_REDIRECT_URI="https://jishnupg-opencode-cli.hf.space/callback"
    export ADMIN_PASSWORD="$INITIAL_PASSWORD"
    export OMNIROUTE_INITIAL_PASSWORD="$INITIAL_PASSWORD"
    export OMNIROUTE_PASSWORD="$INITIAL_PASSWORD"
    export PASSWORD="$INITIAL_PASSWORD"
    export RESET_PASSWORD="$INITIAL_PASSWORD"
    export OMNIROUTE_RESET_PASSWORD="$INITIAL_PASSWORD"
    export CLI_COMPAT_ANTIGRAVITY=1
    export CLI_COMPAT_GITHUB=1
    export CLI_COMPAT_KIMI_CODING=1
    export CLI_COMPAT_CLAUDE=1
    export CLI_COMPAT_CODEX=1
    export CLI_COMPAT_CURSOR=1
    export CLI_COMPAT_QWEN=1
    export OMNIROUTE_AUTO_FREE_FALLBACK_TO_FULL_POOL=true
    export OMNIROUTE_ALLOW_UNAUTHENTICATED=true
    export REQUIRE_API_KEY=false
    export OMNIROUTE_REQUIRE_API_KEY=false
    export OMNIROUTE_AUTH_REQUIRED=false
    export RATE_LIMIT_ENABLED=false
    export ENABLE_RATE_LIMIT=false
    export ENABLE_RATE_LIMITING=false
    export OMNIROUTE_RATE_LIMIT_ENABLED=false
    export AUTH_RATE_LIMIT=false
    export LOGIN_RATE_LIMIT=false
    export DISABLE_RATE_LIMIT=true
    export DISABLE_RATE_LIMITING=true
    export AUTH_RATE_LIMIT_MAX=100000
    export AUTH_RATE_LIMIT_WINDOW_MS=1000
    export RATE_LIMIT_MAX=100000
    export RATE_LIMIT_WINDOW_MS=1000
    export TRUST_PROXY=true

    # ── Master Provider API Keys & Secrets Auto-Discovery ─────────────────────
    export OPENAI_API_KEY="${OPENAI_API_KEY:-}"
    export ANTHROPIC_API_KEY="${ANTHROPIC_API_KEY:-${CLAUDE_API_KEY:-}}"
    export GEMINI_API_KEY="${GEMINI_API_KEY:-${GOOGLE_API_KEY:-${GOOGLE_GENAI_API_KEY:-}}}"
    export DEEPSEEK_API_KEY="${DEEPSEEK_API_KEY:-}"
    export GROQ_API_KEY="${GROQ_API_KEY:-}"
    export NVIDIA_API_KEY="${NVIDIA_API_KEY:-${NVIDIA_NIM_API_KEY:-}}"
    export HUGGINGFACE_API_KEY="${HUGGINGFACE_API_KEY:-${HF_TOKEN:-${HUGGING_FACE_HUB_TOKEN:-}}}"
    export HF_TOKEN="${HF_TOKEN:-$HUGGINGFACE_API_KEY}"
    export OPENROUTER_API_KEY="${OPENROUTER_API_KEY:-}"
    export MISTRAL_API_KEY="${MISTRAL_API_KEY:-}"
    export COHERE_API_KEY="${COHERE_API_KEY:-}"
    export MOONSHOT_API_KEY="${MOONSHOT_API_KEY:-${KIMI_API_KEY:-}}"
    export TOGETHER_API_KEY="${TOGETHER_API_KEY:-}"
    export FIREWORKS_API_KEY="${FIREWORKS_API_KEY:-}"
    export CEREBRAS_API_KEY="${CEREBRAS_API_KEY:-}"
    export PERPLEXITY_API_KEY="${PERPLEXITY_API_KEY:-}"
    export XAI_API_KEY="${XAI_API_KEY:-}"
    export GITHUB_TOKEN="${GITHUB_TOKEN:-${GH_TOKEN:-}}"
    export ANTIGRAVITY_API_KEY="${ANTIGRAVITY_API_KEY:-${ANTIGRAVITY_TOKEN:-}}"

    # ── Restore Persistent Credential Vault into Environment ──────────────────
    if [ -f "/data/omniroute/credentials_vault.json" ]; then
        echo "[VAULT] Restoring persistent provider secrets from /data/omniroute/credentials_vault.json..."
        python3 -c "
import json
try:
    with open('/data/omniroute/credentials_vault.json', 'r', encoding='utf-8') as f:
        v = json.load(f)
    for k, val in v.items():
        if k and val:
            print(f'export {k}=\"{val}\"')
except Exception:
    pass
" > /tmp/vault_env.sh 2>/dev/null || true
        if [ -f "/tmp/vault_env.sh" ]; then
            . /tmp/vault_env.sh
            rm -f /tmp/vault_env.sh
        fi
    fi

    if [ -d "/omniroute" ]; then
        cd /omniroute
        if [ -f "/fix_omniroute.py" ]; then
            python3 /fix_omniroute.py /omniroute 2>&1 || true
        elif [ -f "fix_omniroute.py" ]; then
            python3 fix_omniroute.py ./ 2>&1 || true
        fi
        mkdir -p /data/omniroute 2>/dev/null || true
        if [ -f "server.js" ]; then
            echo "[INIT] OmniRoute entry: node server.js (PORT=20128)"
            node server.js > /data/omniroute/omniroute.log 2>&1 &
        else
            echo "[INIT] OmniRoute entry: npm run start (PORT=20128)"
            npm run start > /data/omniroute/omniroute.log 2>&1 &
        fi
        OMNIROUTE_PID=$!
        (
            for i in $(seq 1 90); do
                if curl -fsS "http://127.0.0.1:20128/api/monitoring/health" >/dev/null 2>&1; then
                    echo "[HEALTH] OmniRoute ready after ${i}s"
                    echo "[PROCESS] OmniRoute: PID ${OMNIROUTE_PID}"
                    break
                fi
                sleep 1
            done
        ) &
    fi

# Step 9: Start Database & Disk Health Doctor Daemon
if [ -f "/health_doctor.py" ]; then
    echo "[HEALTH] Starting Database & Disk Health Doctor daemon..."
    python3 /health_doctor.py > /data/cache/health_doctor.log 2>&1 &
fi

# Step 10: Keep PID 1 Alive and Monitor Child Processes
echo "[BOOT] All services dispatched. Process Supervisor active."
BACKUP_TIMER=0

while true; do
    if [ -n "$FASTAPI_PID" ] && ! kill -0 $FASTAPI_PID 2>/dev/null; then
        echo "[CRITICAL] FastAPI Gateway process died! Restarting..."
        python3 -m uvicorn proxy:app --host 127.0.0.1 --port 8000 --workers 2 --no-access-log --log-level warning > /data/cache/fastapi_gateway.log 2>&1 &
        FASTAPI_PID=$!
    fi

    if [ -n "$NGINX_PID" ] && ! kill -0 $NGINX_PID 2>/dev/null; then
        echo "[CRITICAL] Nginx process died! Restarting..."
        nginx -g 'daemon off;' -c /nginx.conf &
        NGINX_PID=$!
    fi

    if [ -n "$OMNIROUTE_PID" ] && ! kill -0 $OMNIROUTE_PID 2>/dev/null; then
        echo "[CRITICAL] OmniRoute process died! (backoff: ${OMNIROUTE_BACKOFF_S:-2}s) — last crash log:"
        tail -n 20 /data/omniroute/omniroute.log 2>/dev/null || true
        sleep "${OMNIROUTE_BACKOFF_S:-2}"
        (cd /omniroute && node server.js) > /data/omniroute/omniroute.log 2>&1 &
        OMNIROUTE_PID=$!
        # Exponential backoff: 2 → 4 → 8 → 16 → 32 → 60 (cap)
        OMNIROUTE_BACKOFF_S=$(( ${OMNIROUTE_BACKOFF_S:-2} * 2 ))
        if [ "$OMNIROUTE_BACKOFF_S" -gt 60 ]; then OMNIROUTE_BACKOFF_S=60; fi
        OMNIROUTE_FAIL_COUNT=0
    elif [ -n "$OMNIROUTE_PID" ]; then
        _STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 3 "http://127.0.0.1:20128/api/monitoring/health" 2>/dev/null || echo "000")
        if [ "$_STATUS" = "500" ] || [ "$_STATUS" = "000" ]; then
            OMNIROUTE_FAIL_COUNT=$((OMNIROUTE_FAIL_COUNT + 1))
            if [ "$OMNIROUTE_FAIL_COUNT" -ge 12 ]; then
                echo "[SUPERVISOR] OmniRoute unhealthy (HTTP $_STATUS) for 120s. Last crash/server log:"
                tail -n 25 /data/omniroute/omniroute.log 2>/dev/null || true
                echo "[SUPERVISOR] Performing clean self-healing restart..."
                kill -TERM $OMNIROUTE_PID 2>/dev/null || true
                sleep 2
                kill -9 $OMNIROUTE_PID 2>/dev/null || true
                (cd /omniroute && node server.js) > /data/omniroute/omniroute.log 2>&1 &
                OMNIROUTE_PID=$!
                OMNIROUTE_FAIL_COUNT=0
            fi
        else
            OMNIROUTE_FAIL_COUNT=0
            OMNIROUTE_BACKOFF_S=2
        fi
    fi

    # Periodic clean SQLite backup routine every 3600 seconds (1 hour)
    BACKUP_TIMER=$((BACKUP_TIMER + 10))
    if [ "$BACKUP_TIMER" -ge 3600 ]; then
        backup_omniroute_db >/dev/null 2>&1 || true
        BACKUP_TIMER=0
    fi

    sleep 10
done

	#!/usr/bin/env bash
	set -Eeuo pipefail

	START_TIME=$(date +%s)
	FASTAPI_PID=""
	NGINX_PID=""
	OMNIROUTE_PID=""
	OMNIROUTE_FAIL_COUNT=0
	OMNIROUTE_BACKOFF_S=2
	get_elapsed() {
	if [ -n "$START_TIME" ]; then
	python3 -c "import time; print(f'{time.time() - $START_TIME:.2f}s')" 2>/dev/null \|\| echo "0.0s"
	else
	echo "0.0s"
	fi
	}

	log_info() {
	echo "$(date -u '+%Y-%m-%d %H:%M:%S') [INFO] [$1] $2"
	}

	log_warn() {
	echo "$(date -u '+%Y-%m-%d %H:%M:%S') [WARN] [$1] $2"
	}

	log_error() {
	echo "$(date -u '+%Y-%m-%d %H:%M:%S') [ERROR] [$1] $2"
	}

	log_info "SYSTEM" "============================================"
	log_info "SYSTEM" "=== OpenCode Space Container Starting ==="
	log_info "SYSTEM" "============================================"

	# Step 1: Configure Git & Environment Secrets Validation
	git config --global --add safe.directory '*' 2>/dev/null \|\| true

	# Step 2: Ensure persistent /data directory structure exists
	log_info "INIT" "Setting up /data persistent volume directories..."
	mkdir -p /data/share/opencode /data/config/opencode /data/cache/opencode /data/state/opencode 2>/dev/null \|\| true
	mkdir -p /root/.cache /data/cache 2>/dev/null \|\| true

	# Ensure persistent volume directories exist (Never wipe user data on restarts)
	mkdir -p /data/omniroute/backups /data/omniroute/oauth /data/omniroute/credentials /data/omniroute/call_logs /data/omniroute/runtime 2>/dev/null \|\| true
	# Clean only legacy non-OmniRoute directories if lingering
	rm -rf /data/jellyfin /data/hermes /root/.hermes /data/share 2>/dev/null \|\| true
	chmod -R 777 /root/.cache /data/cache /data/omniroute 2>/dev/null \|\| true

	# Playwright Browser Metadata Fix for gemini-web / browser providers
	export PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
	export PLAYWRIGHT_BROWSERS_PATH=/root/.cache/ms-playwright
	mkdir -p /root/.cache/ms-playwright /omniroute/.cache/ms-playwright 2>/dev/null \|\| true
	python3 -c "
	import os, json
	browsers_json = {
	'browsers': [
	{'name': 'chromium', 'revision': '1112', 'installByDefault': True},
	{'name': 'firefox', 'revision': '1448', 'installByDefault': True},
	{'name': 'webkit', 'revision': '2007', 'installByDefault': True},
	{'name': 'ffmpeg', 'revision': '1009', 'installByDefault': True}
	]
	}
	targets = [
	'/root/.cache/ms-playwright',
	'/omniroute/.cache/ms-playwright',
	'/omniroute/node_modules/playwright-core',
	'/omniroute/node_modules/playwright/node_modules/playwright-core',
	'/usr/local/lib/node_modules/playwright-core',
	'/usr/local/lib/node_modules/playwright/node_modules/playwright-core'
	]
	if os.path.exists('/omniroute'):
	for root, dirs, files in os.walk('/omniroute'):
	if os.path.basename(root) == 'playwright-core':
	targets.append(root)

	for target in set(targets):
	try:
	os.makedirs(target, exist_ok=True)
	b_path = os.path.join(target, 'browsers.json')
	if not os.path.exists(b_path) or os.path.getsize(b_path) == 0:
	with open(b_path, 'w', encoding='utf-8') as f:
	json.dump(browsers_json, f, indent=2)
	print(f'[PLAYWRIGHT] Created browsers.json at {b_path}')
	except Exception:
	pass
	" 2>/dev/null \|\| true

	# ── STEP 1: Master Secret Validation & Persistent Encryption Keys ─────────────
	mkdir -p /data/omniroute 2>/dev/null \|\| true

	if [ -n "$STORAGE_ENCRYPTION_KEY" ]; then
	echo "$STORAGE_ENCRYPTION_KEY" > /data/omniroute/.storage_encryption_key 2>/dev/null \|\| true
	elif [ -f "/data/omniroute/.storage_encryption_key" ] && [ -s "/data/omniroute/.storage_encryption_key" ]; then
	export STORAGE_ENCRYPTION_KEY=$(cat /data/omniroute/.storage_encryption_key \| tr -d '\r\n')
	echo "[PERSISTENCE] Loaded persistent STORAGE_ENCRYPTION_KEY from /data/omniroute/.storage_encryption_key"
	else
	export STORAGE_ENCRYPTION_KEY=$(python3 -c "import secrets; print(secrets.token_hex(16))" 2>/dev/null \|\| echo "omniroute_default_storage_key_32bytes")
	echo "$STORAGE_ENCRYPTION_KEY" > /data/omniroute/.storage_encryption_key 2>/dev/null \|\| true
	echo "[PERSISTENCE] Generated and saved persistent STORAGE_ENCRYPTION_KEY to /data/omniroute"
	fi

	if [ -n "$JWT_SECRET" ]; then
	echo "$JWT_SECRET" > /data/omniroute/.jwt_secret 2>/dev/null \|\| true
	elif [ -f "/data/omniroute/.jwt_secret" ] && [ -s "/data/omniroute/.jwt_secret" ]; then
	export JWT_SECRET=$(cat /data/omniroute/.jwt_secret \| tr -d '\r\n')
	echo "[PERSISTENCE] Loaded persistent JWT_SECRET from /data/omniroute/.jwt_secret"
	else
	export JWT_SECRET=$(python3 -c "import secrets; print(secrets.token_hex(16))" 2>/dev/null \|\| echo "omniroute_default_jwt_secret_key_32bytes")
	echo "$JWT_SECRET" > /data/omniroute/.jwt_secret 2>/dev/null \|\| true
	echo "[PERSISTENCE] Generated and saved persistent JWT_SECRET to /data/omniroute"
	fi

	if [ -n "$API_KEY_SECRET" ]; then
	echo "$API_KEY_SECRET" > /data/omniroute/.api_key_secret 2>/dev/null \|\| true
	elif [ -f "/data/omniroute/.api_key_secret" ] && [ -s "/data/omniroute/.api_key_secret" ]; then
	export API_KEY_SECRET=$(cat /data/omniroute/.api_key_secret \| tr -d '\r\n')
	echo "[PERSISTENCE] Loaded persistent API_KEY_SECRET from /data/omniroute/.api_key_secret"
	else
	export API_KEY_SECRET=$(python3 -c "import secrets; print(secrets.token_hex(16))" 2>/dev/null \|\| echo "omniroute_default_api_key_secret_32bytes")
	echo "$API_KEY_SECRET" > /data/omniroute/.api_key_secret 2>/dev/null \|\| true
	echo "[PERSISTENCE] Generated and saved persistent API_KEY_SECRET to /data/omniroute"
	fi

	export INITIAL_PASSWORD="${INITIAL_PASSWORD:-Jishnu2005}"
	echo "$INITIAL_PASSWORD" > /data/omniroute/.initial_password 2>/dev/null \|\| true
	echo "[PERSISTENCE] Configured OmniRoute administrator password to persistent storage."

	export ENCRYPTION_SECRET="${STORAGE_ENCRYPTION_KEY}"
	export OMNIROUTE_SECRET_KEY="${STORAGE_ENCRYPTION_KEY}"
	export OMNIROUTE_STORAGE_KEY="${STORAGE_ENCRYPTION_KEY}"
	export OMNIROUTE_STORAGE_SECRET="${STORAGE_ENCRYPTION_KEY}"
	export OMNIROUTE_WS_BRIDGE_SECRET="${OMNIROUTE_WS_BRIDGE_SECRET:-$(echo "ws_bridge_${JWT_SECRET}" \| sha256sum \| cut -c1-48)}"

	# Purge any legacy secret env files from storage bucket to maintain strict secret separation
	rm -f /data/.env /data/secrets.env /data/secrets /data/config/.env /data/omniroute/.env /data/omniroute/secrets.env /data/omniroute/secrets /data/omniroute/.secrets /data/omniroute/server.env 2>/dev/null \|\| true

	echo "[BOOT] Secrets validated: $(get_elapsed)"

	# ── STEP 2: Canonical Direct Persistent Storage & Preflight Gate ──────────────
	export DATA_DIR="/data/omniroute"
	export OMNIROUTE_DATA_DIR="/data/omniroute"
	export OMNIROUTE_RUNTIME_DIR="/root/.omniroute"
	export OMNIROUTE_RUNTIME_DB="/root/.omniroute/storage.sqlite"
	mkdir -p "$OMNIROUTE_RUNTIME_DIR" "$DATA_DIR/.incoming"

	preflight_persistence_gate() {
	log_info "PREFLIGHT" "Validating persistent storage invariants for OmniRoute..."
	if [ ! -d "/data" ]; then
	log_error "PREFLIGHT" "CRITICAL: /data directory does not exist! Persistent bucket not mounted."
	return 1
	fi
	if [ ! -w "/data" ]; then
	log_error "PREFLIGHT" "CRITICAL: /data is not writable!"
	return 1
	fi
	mkdir -p "$DATA_DIR/backups" "$DATA_DIR/oauth" "$DATA_DIR/credentials" "$DATA_DIR/call_logs" "$DATA_DIR/runtime" 2>/dev/null \|\| true
	if [ ! -w "$DATA_DIR" ]; then
	log_error "PREFLIGHT" "CRITICAL: DATA_DIR ($DATA_DIR) is not writable!"
	return 1
	fi

	# Touch probe to verify read-write access
	_PROBE="$DATA_DIR/.probe_$$"
	if ! touch "$_PROBE" 2>/dev/null; then
	log_error "PREFLIGHT" "CRITICAL: Write probe failed in $DATA_DIR!"
	return 1
	fi
	rm -f "$_PROBE" 2>/dev/null \|\| true

	# Cloud Vault Restore: Synchronize clean, vacuumed authoritative database
	log_info "PREFLIGHT" "Synchronizing 100% verified vacuumed SQLite database from cloud vault..."
	rm -f "$DATA_DIR/storage.sqlite"* "$DATA_DIR/backups/last-known-good.sqlite"* "$OMNIROUTE_RUNTIME_DIR/storage.sqlite"* 2>/dev/null \|\| true
	if [ -f "/vault_sync.py" ]; then
	python3 /vault_sync.py restore --force \|\| true
	fi

	log_info "PREFLIGHT" "Persistence preflight passed successfully. Single source of truth: $DATA_DIR"
	return 0
	}

	preflight_persistence_gate \|\| log_error "PREFLIGHT" "Warning: persistence preflight reported issues, continuing with best effort."

	# Runtime SQLite is authoritative; /data contains only verified snapshots.
	if ! python3 /persistence_manager.py recover 2>&1 \| tee /data/cache/persistence-recovery.log; then
	log_error "PERSISTENCE" "Verified runtime database recovery failed; refusing to start OmniRoute."
	exit 1
	fi

	# SQLite-Aware Backup Routine (Scheduled & On Shutdown)
	backup_omniroute_db() {
	if [ -f "$OMNIROUTE_RUNTIME_DB" ]; then
	if python3 /persistence_manager.py snapshot >> /data/cache/persistence.log 2>&1; then
	log_info "BACKUP" "Verified SQLite snapshot promoted."
	if [ -f "/vault_sync.py" ]; then
	python3 /vault_sync.py backup >/dev/null 2>&1 &
	fi
	else
	log_warn "BACKUP" "Verified snapshot failed; previous persistent snapshot preserved."
	fi
	fi
	}

	shutdown_gracefully() {
	log_info "SHUTDOWN" "Process termination signal received. Flushing persistence state..."
	if [ -n "$OMNIROUTE_PID" ] && kill -0 $OMNIROUTE_PID 2>/dev/null; then
	kill -INT $OMNIROUTE_PID 2>/dev/null \|\| true
	for _i in $(seq 1 10); do
	if ! kill -0 $OMNIROUTE_PID 2>/dev/null; then
	break
	fi
	sleep 1
	done
	fi
	if [ -f "$OMNIROUTE_RUNTIME_DB" ]; then
	backup_omniroute_db >/dev/null 2>&1 \|\| true
	if [ -f "/vault_sync.py" ]; then
	python3 /vault_sync.py backup >/dev/null 2>&1 \|\| true
	fi
	fi
	log_info "SHUTDOWN" "Persistence checkpoint complete. Container exiting."
	exit 0
	}

	trap shutdown_gracefully EXIT INT TERM

	export HF_HUB_DISABLE_PROGRESS_BARS=1
	export TQDM_DISABLE=1

	# ── STEP 3: Start FastAPI Gateway Immediately ────────────────────────────────
	echo "[BOOT] FastAPI starting: $(get_elapsed)"
	cd /
	python3 -m uvicorn proxy:app --host 127.0.0.1 --port 8000 --workers 2 --no-access-log --log-level warning &
	FASTAPI_PID=$!

	# Step 4: Wait for FastAPI /health/live to return HTTP 200
	FASTAPI_READY=0
	for i in $(seq 1 30); do
	if curl --max-time 2 -sf -i http://127.0.0.1:8000/health/live \| grep -q "200 OK"; then
	FASTAPI_READY=1
	END_TIME=$(date +%s)
	ELAPSED=$((END_TIME - START_TIME))
	echo "[BOOT] FastAPI live: ${ELAPSED}s"
	break
	fi
	sleep 0.2
	done

	if [ "$FASTAPI_READY" -ne 1 ]; then
	echo "[WARN] FastAPI did not report ready within timeout, proceeding with startup..."
	fi

	# Step 5: Start Nginx Edge Server on Public Port 4096
	echo "[BOOT] Nginx starting..."
	rm -f /etc/nginx/sites-enabled/default /etc/nginx/sites-available/default 2>/dev/null \|\| true
	nginx -t -c /nginx.conf
	nginx -g 'daemon off;' -c /nginx.conf &
	NGINX_PID=$!

	# Step 6: Wait for Nginx /health/live on Port 4096 to return HTTP 200
	NGINX_READY=0
	for i in $(seq 1 30); do
	if curl --max-time 2 -sf -i http://127.0.0.1:4096/health/live \| grep -q "200 OK"; then
	NGINX_READY=1
	break
	fi
	sleep 0.2
	done

	# Step 7: Signal Public Gateway Readiness immediately to Hugging Face
	echo "[BOOT] Public gateway live"
	echo "[BOOT] Background services starting asynchronously..."

	# Step 8: Start Redis Server & OmniRoute AI Gateway in Background
	if command -v redis-server >/dev/null 2>&1; then
	echo "[INIT] Starting Redis server on port 6379..."
	redis-server --daemonize yes 2>/dev/null \|\| true
	if command -v redis-cli >/dev/null 2>&1; then
	echo "[INIT] Redis server active on port 6379."
	fi
	fi

	echo "[INIT] Starting OmniRoute AI Gateway..."
	export PORT=20128
	export HOSTNAME="127.0.0.1"
	export DATA_DIR="$OMNIROUTE_RUNTIME_DIR"
	export OMNIROUTE_DATA_DIR="$OMNIROUTE_RUNTIME_DIR"
	export REDIS_URL="redis://127.0.0.1:6379"
	export NEXT_PUBLIC_BASE_URL="https://jishnupg-opencode-cli.hf.space"
	export AUTH_COOKIE_SECURE="true"
	export ALLOW_REMOTE_OAUTH="true"
	export ALLOW_BROWSER_OAUTH="true"
	export OAUTH_CALLBACK_URL="https://jishnupg-opencode-cli.hf.space/callback"
	export NEXT_PUBLIC_OAUTH_CALLBACK_URL="https://jishnupg-opencode-cli.hf.space/callback"
	export REDIRECT_URI="https://jishnupg-opencode-cli.hf.space/callback"
	export ANTIGRAVITY_OAUTH_REDIRECT_URI="https://jishnupg-opencode-cli.hf.space/callback"
	export ANTIGRAVITY_REDIRECT_URI="https://jishnupg-opencode-cli.hf.space/callback"
	export ADMIN_PASSWORD="$INITIAL_PASSWORD"
	export OMNIROUTE_INITIAL_PASSWORD="$INITIAL_PASSWORD"
	export OMNIROUTE_PASSWORD="$INITIAL_PASSWORD"
	export PASSWORD="$INITIAL_PASSWORD"
	export RESET_PASSWORD="$INITIAL_PASSWORD"
	export OMNIROUTE_RESET_PASSWORD="$INITIAL_PASSWORD"
	export CLI_COMPAT_ANTIGRAVITY=1
	export CLI_COMPAT_GITHUB=1
	export CLI_COMPAT_KIMI_CODING=1
	export CLI_COMPAT_CLAUDE=1
	export CLI_COMPAT_CODEX=1
	export CLI_COMPAT_CURSOR=1
	export CLI_COMPAT_QWEN=1
	export OMNIROUTE_AUTO_FREE_FALLBACK_TO_FULL_POOL=true
	export OMNIROUTE_ALLOW_UNAUTHENTICATED=true
	export REQUIRE_API_KEY=false
	export OMNIROUTE_REQUIRE_API_KEY=false
	export OMNIROUTE_AUTH_REQUIRED=false
	export RATE_LIMIT_ENABLED=false
	export ENABLE_RATE_LIMIT=false
	export ENABLE_RATE_LIMITING=false
	export OMNIROUTE_RATE_LIMIT_ENABLED=false
	export AUTH_RATE_LIMIT=false
	export LOGIN_RATE_LIMIT=false
	export DISABLE_RATE_LIMIT=true
	export DISABLE_RATE_LIMITING=true
	export AUTH_RATE_LIMIT_MAX=100000
	export AUTH_RATE_LIMIT_WINDOW_MS=1000
	export RATE_LIMIT_MAX=100000
	export RATE_LIMIT_WINDOW_MS=1000
	export TRUST_PROXY=true

	# ── Master Provider API Keys & Secrets Auto-Discovery ─────────────────────
	export OPENAI_API_KEY="${OPENAI_API_KEY:-}"
	export ANTHROPIC_API_KEY="${ANTHROPIC_API_KEY:-${CLAUDE_API_KEY:-}}"
	export GEMINI_API_KEY="${GEMINI_API_KEY:-${GOOGLE_API_KEY:-${GOOGLE_GENAI_API_KEY:-}}}"
	export DEEPSEEK_API_KEY="${DEEPSEEK_API_KEY:-}"
	export GROQ_API_KEY="${GROQ_API_KEY:-}"
	export NVIDIA_API_KEY="${NVIDIA_API_KEY:-${NVIDIA_NIM_API_KEY:-}}"
	export HUGGINGFACE_API_KEY="${HUGGINGFACE_API_KEY:-${HF_TOKEN:-${HUGGING_FACE_HUB_TOKEN:-}}}"
	export HF_TOKEN="${HF_TOKEN:-$HUGGINGFACE_API_KEY}"
	export OPENROUTER_API_KEY="${OPENROUTER_API_KEY:-}"
	export MISTRAL_API_KEY="${MISTRAL_API_KEY:-}"
	export COHERE_API_KEY="${COHERE_API_KEY:-}"
	export MOONSHOT_API_KEY="${MOONSHOT_API_KEY:-${KIMI_API_KEY:-}}"
	export TOGETHER_API_KEY="${TOGETHER_API_KEY:-}"
	export FIREWORKS_API_KEY="${FIREWORKS_API_KEY:-}"
	export CEREBRAS_API_KEY="${CEREBRAS_API_KEY:-}"
	export PERPLEXITY_API_KEY="${PERPLEXITY_API_KEY:-}"
	export XAI_API_KEY="${XAI_API_KEY:-}"
	export GITHUB_TOKEN="${GITHUB_TOKEN:-${GH_TOKEN:-}}"
	export ANTIGRAVITY_API_KEY="${ANTIGRAVITY_API_KEY:-${ANTIGRAVITY_TOKEN:-}}"

	# ── Restore Persistent Credential Vault into Environment ──────────────────
	if [ -f "/data/omniroute/credentials_vault.json" ]; then
	echo "[VAULT] Restoring persistent provider secrets from /data/omniroute/credentials_vault.json..."
	python3 -c "
	import json
	try:
	with open('/data/omniroute/credentials_vault.json', 'r', encoding='utf-8') as f:
	v = json.load(f)
	for k, val in v.items():
	if k and val:
	print(f'export {k}=\"{val}\"')
	except Exception:
	pass
	" > /tmp/vault_env.sh 2>/dev/null \|\| true
	if [ -f "/tmp/vault_env.sh" ]; then
	. /tmp/vault_env.sh
	rm -f /tmp/vault_env.sh
	fi
	fi

	if [ -d "/omniroute" ]; then
	cd /omniroute
	if [ -f "/fix_omniroute.py" ]; then
	python3 /fix_omniroute.py /omniroute 2>&1 \|\| true
	elif [ -f "fix_omniroute.py" ]; then
	python3 fix_omniroute.py ./ 2>&1 \|\| true
	fi
	mkdir -p /data/omniroute 2>/dev/null \|\| true
	if [ -f "server.js" ]; then
	echo "[INIT] OmniRoute entry: node server.js (PORT=20128)"
	node server.js > /data/omniroute/omniroute.log 2>&1 &
	else
	echo "[INIT] OmniRoute entry: npm run start (PORT=20128)"
	npm run start > /data/omniroute/omniroute.log 2>&1 &
	fi
	OMNIROUTE_PID=$!
	(
	for i in $(seq 1 90); do
	if curl -fsS "http://127.0.0.1:20128/api/monitoring/health" >/dev/null 2>&1; then
	echo "[HEALTH] OmniRoute ready after ${i}s"
	echo "[PROCESS] OmniRoute: PID ${OMNIROUTE_PID}"
	break
	fi
	sleep 1
	done
	) &
	fi

	# Step 9: Start Database & Disk Health Doctor Daemon
	if [ -f "/health_doctor.py" ]; then
	echo "[HEALTH] Starting Database & Disk Health Doctor daemon..."
	python3 /health_doctor.py > /data/cache/health_doctor.log 2>&1 &
	fi

	# Step 10: Keep PID 1 Alive and Monitor Child Processes
	echo "[BOOT] All services dispatched. Process Supervisor active."
	BACKUP_TIMER=0

	while true; do
	if [ -n "$FASTAPI_PID" ] && ! kill -0 $FASTAPI_PID 2>/dev/null; then
	echo "[CRITICAL] FastAPI Gateway process died! Restarting..."
	python3 -m uvicorn proxy:app --host 127.0.0.1 --port 8000 --workers 2 --no-access-log --log-level warning > /data/cache/fastapi_gateway.log 2>&1 &
	FASTAPI_PID=$!
	fi

	if [ -n "$NGINX_PID" ] && ! kill -0 $NGINX_PID 2>/dev/null; then
	echo "[CRITICAL] Nginx process died! Restarting..."
	nginx -g 'daemon off;' -c /nginx.conf &
	NGINX_PID=$!
	fi

	if [ -n "$OMNIROUTE_PID" ] && ! kill -0 $OMNIROUTE_PID 2>/dev/null; then
	echo "[CRITICAL] OmniRoute process died! (backoff: ${OMNIROUTE_BACKOFF_S:-2}s) — last crash log:"
	tail -n 20 /data/omniroute/omniroute.log 2>/dev/null \|\| true
	sleep "${OMNIROUTE_BACKOFF_S:-2}"
	(cd /omniroute && node server.js) > /data/omniroute/omniroute.log 2>&1 &
	OMNIROUTE_PID=$!
	# Exponential backoff: 2 → 4 → 8 → 16 → 32 → 60 (cap)
	OMNIROUTE_BACKOFF_S=$(( ${OMNIROUTE_BACKOFF_S:-2} * 2 ))
	if [ "$OMNIROUTE_BACKOFF_S" -gt 60 ]; then OMNIROUTE_BACKOFF_S=60; fi
	OMNIROUTE_FAIL_COUNT=0
	elif [ -n "$OMNIROUTE_PID" ]; then
	_STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 3 "http://127.0.0.1:20128/api/monitoring/health" 2>/dev/null \|\| echo "000")
	if [ "$_STATUS" = "500" ] \|\| [ "$_STATUS" = "000" ]; then
	OMNIROUTE_FAIL_COUNT=$((OMNIROUTE_FAIL_COUNT + 1))
	if [ "$OMNIROUTE_FAIL_COUNT" -ge 12 ]; then
	echo "[SUPERVISOR] OmniRoute unhealthy (HTTP $_STATUS) for 120s. Last crash/server log:"
	tail -n 25 /data/omniroute/omniroute.log 2>/dev/null \|\| true
	echo "[SUPERVISOR] Performing clean self-healing restart..."
	kill -TERM $OMNIROUTE_PID 2>/dev/null \|\| true
	sleep 2
	kill -9 $OMNIROUTE_PID 2>/dev/null \|\| true
	(cd /omniroute && node server.js) > /data/omniroute/omniroute.log 2>&1 &
	OMNIROUTE_PID=$!
	OMNIROUTE_FAIL_COUNT=0
	fi
	else
	OMNIROUTE_FAIL_COUNT=0
	OMNIROUTE_BACKOFF_S=2
	fi
	fi

	# Periodic clean SQLite backup routine every 3600 seconds (1 hour)
	BACKUP_TIMER=$((BACKUP_TIMER + 10))
	if [ "$BACKUP_TIMER" -ge 3600 ]; then
	backup_omniroute_db >/dev/null 2>&1 \|\| true
	BACKUP_TIMER=0
	fi

	sleep 10
	done