start.sh

#!/bin/sh
set -eu
umask 077

log() {
  printf '[runtime-hf] %s\n' "$*" >&2
}

fail() {
  log "ERROR: $*"
  exit 1
}

yaml_escape() {
  printf "%s" "$1" | sed "s/'/''/g"
}

sql_escape() {
  printf "%s" "$1" | sed "s/'/''/g"
}

if [ -z "${DATA_DIR:-}" ] && [ -d /data ]; then
  export DATA_DIR=/data/appdata
else
  export DATA_DIR="${DATA_DIR:-/app/data}"
fi

export AUTO_SETUP="${AUTO_SETUP:-true}"
export SERVER_HOST=0.0.0.0
export SERVER_PORT="${SERVER_PORT:-8080}"
export SERVER_MODE="${SERVER_MODE:-release}"
export RUN_MODE="${RUN_MODE:-standard}"
export TZ="${TZ:-Asia/Shanghai}"

# Production mode: run PostgreSQL inside this Space on ephemeral disk and
# archive encrypted WAL/base/logical backups to the mounted HF Storage Bucket.
export LOCAL_POSTGRES_ENABLED="${LOCAL_POSTGRES_ENABLED:-true}"

if [ -z "${BACKUP_ROOT:-}" ]; then
  export BACKUP_ROOT="/data/sub2api-pg-backups"
fi

export LOCAL_PGDATA="${LOCAL_PGDATA:-/tmp/app-pgdata}"
export LOCAL_DATABASE_HOST="127.0.0.1"
export LOCAL_DATABASE_PORT="${LOCAL_DATABASE_PORT:-5432}"
export LOCAL_DATABASE_USER="${LOCAL_DATABASE_USER:-appuser}"
export LOCAL_DATABASE_DBNAME="${LOCAL_DATABASE_DBNAME:-appdb}"
export LOCAL_DATABASE_PASSWORD="${LOCAL_DATABASE_PASSWORD:-${DATABASE_PASSWORD:-}}"
export BACKUP_ENCRYPTION_KEY="${BACKUP_ENCRYPTION_KEY:-${DATABASE_PASSWORD:-}}"
export LOCAL_PG_ARCHIVE_TIMEOUT="${LOCAL_PG_ARCHIVE_TIMEOUT:-30s}"
export LOCAL_PG_PROMOTE_WAIT_SECONDS="${LOCAL_PG_PROMOTE_WAIT_SECONDS:-900}"
export BASE_BACKUP_INTERVAL_SECONDS="${BASE_BACKUP_INTERVAL_SECONDS:-21600}"
export LOGICAL_DUMP_INTERVAL_SECONDS="${LOGICAL_DUMP_INTERVAL_SECONDS:-600}"
export BACKUP_CLEANUP_ENABLED="${BACKUP_CLEANUP_ENABLED:-true}"
export BACKUP_CLEANUP_INTERVAL_SECONDS="${BACKUP_CLEANUP_INTERVAL_SECONDS:-3600}"
export LOCAL_DUMP_RETENTION_HOURS="${LOCAL_DUMP_RETENTION_HOURS:-24}"
export BASE_BACKUP_RETAIN_COUNT="${BASE_BACKUP_RETAIN_COUNT:-4}"
export WAL_RETENTION_HOURS="${WAL_RETENTION_HOURS:-4}"
export STARTUP_RESTORE_STRATEGY="${STARTUP_RESTORE_STRATEGY:-logical_then_base}"

file_mtime_epoch() {
  stat -c %Y "$1" 2>/dev/null || stat -f %m "$1" 2>/dev/null || printf '0\n'
}

# Conservative DB pool profile for HF cpu-basic: keep app-side pool below
# PostgreSQL max_connections, leaving headroom for maintenance, backups and psql.
export DATABASE_MAX_OPEN_CONNS="${DATABASE_MAX_OPEN_CONNS:-160}"
export DATABASE_MAX_IDLE_CONNS="${DATABASE_MAX_IDLE_CONNS:-60}"
export DATABASE_CONN_MAX_LIFETIME_MINUTES="${DATABASE_CONN_MAX_LIFETIME_MINUTES:-30}"
export DATABASE_CONN_MAX_IDLE_TIME_MINUTES="${DATABASE_CONN_MAX_IDLE_TIME_MINUTES:-5}"

write_runtime_config() {
  config_file="${DATA_DIR}/config.yaml"
  install_lock="${DATA_DIR}/.installed"

  user_concurrency=5
  if [ "${RUN_MODE}" = "simple" ]; then
    user_concurrency=30
  fi

  if [ -z "${JWT_SECRET:-}" ]; then
    fail "JWT_SECRET is required when bootstrapping from env on Hugging Face"
  fi

  db_host="${DATABASE_HOST}"
  db_port="${DATABASE_PORT}"
  db_user="${DATABASE_USER}"
  db_password="${DATABASE_PASSWORD}"
  db_name="${DATABASE_DBNAME}"
  db_sslmode="${DATABASE_SSLMODE:-require}"

  if [ "${LOCAL_POSTGRES_ENABLED}" = "true" ]; then
    db_host="${LOCAL_DATABASE_HOST}"
    db_port="${LOCAL_DATABASE_PORT}"
    db_user="${LOCAL_DATABASE_USER}"
    db_password="${LOCAL_DATABASE_PASSWORD}"
    db_name="${LOCAL_DATABASE_DBNAME}"
    db_sslmode="disable"
  fi

  cat > "${config_file}" <<EOCFG
server:
  host: '$(yaml_escape "${SERVER_HOST}")'
  port: ${SERVER_PORT}
  mode: '$(yaml_escape "${SERVER_MODE}")'
database:
  host: '$(yaml_escape "${db_host}")'
  port: ${db_port}
  user: '$(yaml_escape "${db_user}")'
  password: '$(yaml_escape "${db_password}")'
  dbname: '$(yaml_escape "${db_name}")'
  sslmode: '$(yaml_escape "${db_sslmode}")'
  max_open_conns: ${DATABASE_MAX_OPEN_CONNS}
  max_idle_conns: ${DATABASE_MAX_IDLE_CONNS}
  conn_max_lifetime_minutes: ${DATABASE_CONN_MAX_LIFETIME_MINUTES}
  conn_max_idle_time_minutes: ${DATABASE_CONN_MAX_IDLE_TIME_MINUTES}
redis:
  host: '$(yaml_escape "${REDIS_HOST}")'
  port: ${REDIS_PORT}
  password: '$(yaml_escape "${REDIS_PASSWORD:-}")'
  db: ${REDIS_DB}
  enable_tls: ${REDIS_ENABLE_TLS}
jwt:
  secret: '$(yaml_escape "${JWT_SECRET}")'
  expire_hour: ${JWT_EXPIRE_HOUR:-24}
default:
  user_concurrency: ${user_concurrency}
  user_balance: 0
  api_key_prefix: sk-
  rate_multiplier: 1
rate_limit:
  requests_per_minute: 60
  burst_size: 10
timezone: '$(yaml_escape "${TZ}")'
EOCFG

  chmod 600 "${config_file}"
  printf 'installed_at=%s\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" > "${install_lock}"
  chmod 400 "${install_lock}"
}

ensure_bucket_layout() {
  [ -d /data ] || fail "HF Storage Bucket is not mounted at /data"
  [ -w /data ] || fail "HF Storage Bucket mount /data is not writable"
  mkdir -p \
    "${BACKUP_ROOT}/base" \
    "${BACKUP_ROOT}/wal" \
    "${BACKUP_ROOT}/dumps" \
    "${BACKUP_ROOT}/state" \
    "${BACKUP_ROOT}/tmp"
  # archive_command runs as the postgres user, so the mounted bucket subtree
  # must be writable by postgres. HF bucket mounts support chown in this Space.
  chown -R postgres:postgres "${BACKUP_ROOT}" 2>/dev/null || true
}

write_pg_helper_scripts() {
  cat > /app/archive_wal.sh <<'EOS'
#!/bin/sh
set -eu
src="$1"
name="$2"
root="${BACKUP_ROOT:?BACKUP_ROOT required}"
: "${BACKUP_ENCRYPTION_KEY:?BACKUP_ENCRYPTION_KEY required}"
mkdir -p "${root}/wal" "${root}/tmp"
out="${root}/wal/${name}.enc"
[ -s "${out}" ] && exit 0
tmp="${root}/tmp/${name}.enc.$$"
openssl enc -aes-256-cbc -salt -pbkdf2 -pass env:BACKUP_ENCRYPTION_KEY -in "${src}" -out "${tmp}"
mv "${tmp}" "${out}"
EOS

  cat > /app/restore_wal.sh <<'EOS'
#!/bin/sh
set -eu
name="$1"
dest="$2"
root="${BACKUP_ROOT:?BACKUP_ROOT required}"
: "${BACKUP_ENCRYPTION_KEY:?BACKUP_ENCRYPTION_KEY required}"
infile="${root}/wal/${name}.enc"
[ -s "${infile}" ] || exit 1
openssl enc -d -aes-256-cbc -pbkdf2 -pass env:BACKUP_ENCRYPTION_KEY -in "${infile}" -out "${dest}"
EOS

  chmod 755 /app/archive_wal.sh /app/restore_wal.sh
}

configure_pg_for_archive() {
  conf="${LOCAL_PGDATA}/postgresql.conf"
  hba="${LOCAL_PGDATA}/pg_hba.conf"
  marker="# runtime-hf-local-pg"
  if ! grep -q "${marker}" "${conf}" 2>/dev/null; then
    cat >> "${conf}" <<EOCONF

${marker}
listen_addresses = '127.0.0.1'
port = ${LOCAL_DATABASE_PORT}
unix_socket_directories = '/tmp'
wal_level = replica
archive_mode = on
archive_timeout = '${LOCAL_PG_ARCHIVE_TIMEOUT}'
archive_command = '/app/archive_wal.sh "%p" "%f"'
max_wal_size = '1GB'
checkpoint_timeout = '5min'
EOCONF
  fi

  tuning_marker="# runtime-hf-pg-tuning-v2"
  if ! grep -q "${tuning_marker}" "${conf}" 2>/dev/null; then
    cat >> "${conf}" <<EOCONF

${tuning_marker}
max_connections = 200
shared_buffers = '1GB'
effective_cache_size = '6GB'
work_mem = '8MB'
maintenance_work_mem = '256MB'
max_wal_size = '2GB'
checkpoint_timeout = '10min'
wal_compression = on
EOCONF
  fi

  if ! grep -q "runtime-hf-local-pg" "${hba}" 2>/dev/null; then
    cat >> "${hba}" <<EOHBA

# runtime-hf-local-pg
local   all             all                                     trust
host    all             all             127.0.0.1/32            md5
host    replication     all             127.0.0.1/32            trust
EOHBA
  fi
}

restore_latest_base_backup() {
  latest="$(ls -1 "${BACKUP_ROOT}"/base/base-*.tar.gz.enc 2>/dev/null | sort | tail -1 || true)"
  [ -n "${latest}" ] || return 1
  log "restoring local PostgreSQL from encrypted base backup ${latest}"
  rm -rf "${LOCAL_PGDATA}"
  mkdir -p "${LOCAL_PGDATA}"
  tmp="/tmp/app-base-restore.tar.gz"
  openssl enc -d -aes-256-cbc -pbkdf2 -pass env:BACKUP_ENCRYPTION_KEY -in "${latest}" -out "${tmp}"
  tar -xzf "${tmp}" -C "${LOCAL_PGDATA}"
  rm -f "${tmp}" "${LOCAL_PGDATA}/postmaster.pid"
  configure_pg_for_archive
  cat >> "${LOCAL_PGDATA}/postgresql.conf" <<'EOCONF'
restore_command = '/app/restore_wal.sh "%f" "%p"'
EOCONF
  touch "${LOCAL_PGDATA}/recovery.signal"
  touch "${LOCAL_PGDATA}/.app_ready"
  chown postgres:postgres "${LOCAL_PGDATA}/.app_ready"
  chown -R postgres:postgres "${LOCAL_PGDATA}"
  return 0
}

start_postgres() {
  log "starting local PostgreSQL on 127.0.0.1:${LOCAL_DATABASE_PORT}"
  chown -R postgres:postgres "${LOCAL_PGDATA}"
  rm -f /tmp/app-postgres.log
  if ! su-exec postgres pg_ctl -D "${LOCAL_PGDATA}" -l /tmp/app-postgres.log -w start; then
    log "PostgreSQL failed to start; log follows"
    cat /tmp/app-postgres.log 2>/dev/null || true
    return 1
  fi
}

promote_if_in_recovery() {
  if [ "$(su-exec postgres psql -At -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U postgres -d postgres -c 'SELECT pg_is_in_recovery();')" = "t" ]; then
    log "promoting restored PostgreSQL out of recovery"
    su-exec postgres pg_ctl -D "${LOCAL_PGDATA}" promote || log "pg_ctl promote was not needed or not supported; waiting for recovery to finish"
    for _ in $(seq 1 "${LOCAL_PG_PROMOTE_WAIT_SECONDS}"); do
      if [ "$(su-exec postgres psql -At -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U postgres -d postgres -c 'SELECT pg_is_in_recovery();')" = "f" ]; then
        log "restored PostgreSQL is writable"
        return 0
      fi
      sleep 1
    done
    fail "restored PostgreSQL is still in recovery after waiting"
  fi
}

ensure_local_role_db() {
  su-exec postgres psql -v ON_ERROR_STOP=1 -v role_password="${LOCAL_DATABASE_PASSWORD}" -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U postgres -d postgres <<EOSQL
SELECT 'CREATE ROLE ${LOCAL_DATABASE_USER} LOGIN SUPERUSER'
WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${LOCAL_DATABASE_USER}')\gexec
ALTER ROLE ${LOCAL_DATABASE_USER} WITH LOGIN SUPERUSER PASSWORD :'role_password';
SELECT 'CREATE DATABASE ${LOCAL_DATABASE_DBNAME} OWNER ${LOCAL_DATABASE_USER}'
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '${LOCAL_DATABASE_DBNAME}')\gexec
EOSQL
}

restore_latest_logical_dump() {
  latest="$(ls -1 "${BACKUP_ROOT}"/dumps/local-*.dump.enc 2>/dev/null | sort | tail -1 || true)"
  [ -n "${latest}" ] || return 1

  log "restoring local PostgreSQL from encrypted logical dump ${latest}"
  rm -rf "${LOCAL_PGDATA}"
  mkdir -p "${LOCAL_PGDATA}"
  chown -R postgres:postgres "${LOCAL_PGDATA}"
  su-exec postgres initdb -D "${LOCAL_PGDATA}" --encoding=UTF8 --locale=C
  configure_pg_for_archive
  start_postgres
  promote_if_in_recovery
  ensure_local_role_db

  dump="/tmp/runtime-latest-local.dump"
  rm -f "${dump}"
  openssl enc -d -aes-256-cbc -pbkdf2 -pass env:BACKUP_ENCRYPTION_KEY -in "${latest}" -out "${dump}"

  su-exec postgres psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U postgres -d postgres <<EOSQL
SELECT pg_terminate_backend(pid)
FROM pg_stat_activity
WHERE datname = '${LOCAL_DATABASE_DBNAME}'
  AND pid <> pg_backend_pid();
DROP DATABASE IF EXISTS ${LOCAL_DATABASE_DBNAME} WITH (FORCE);
CREATE DATABASE ${LOCAL_DATABASE_DBNAME} OWNER ${LOCAL_DATABASE_USER};
EOSQL

  PGPASSWORD="${LOCAL_DATABASE_PASSWORD}" \
    pg_restore -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U "${LOCAL_DATABASE_USER}" -d "${LOCAL_DATABASE_DBNAME}" \
      --no-owner --no-privileges --exit-on-error "${dump}"

  rm -f "${dump}"
  touch "${LOCAL_PGDATA}/.app_ready"
  chown postgres:postgres "${LOCAL_PGDATA}/.app_ready"
  return 0
}

make_base_backup() {
  ts="$(date -u +%Y%m%dT%H%M%SZ)"
  tmpdir="/tmp/base-${ts}"
  tarball="/tmp/base-${ts}.tar.gz"
  outfile="${BACKUP_ROOT}/base/base-${ts}.tar.gz.enc"
  rm -rf "${tmpdir}" "${tarball}"
  mkdir -p "${tmpdir}"
  chown postgres:postgres "${tmpdir}"
  log "creating encrypted physical base backup ${outfile}"
  su-exec postgres pg_basebackup -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U postgres -D "${tmpdir}" -Fp -Xs
  tar -C "${tmpdir}" -czf "${tarball}" .
  openssl enc -aes-256-cbc -salt -pbkdf2 -pass env:BACKUP_ENCRYPTION_KEY -in "${tarball}" -out "${outfile}"
  rm -rf "${tmpdir}" "${tarball}"
  printf '%s\n' "${outfile}" > "${BACKUP_ROOT}/state/latest-base.txt"
}

make_logical_dump() {
  ts="$(date -u +%Y%m%dT%H%M%SZ)"
  dump="/tmp/local-${ts}.dump"
  outfile="${BACKUP_ROOT}/dumps/local-${ts}.dump.enc"
  log "creating encrypted logical dump ${outfile}"
  PGPASSWORD="${LOCAL_DATABASE_PASSWORD}" \
    pg_dump -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U "${LOCAL_DATABASE_USER}" -d "${LOCAL_DATABASE_DBNAME}" \
      -Fc --no-owner --no-privileges -f "${dump}"
  openssl enc -aes-256-cbc -salt -pbkdf2 -pass env:BACKUP_ENCRYPTION_KEY -in "${dump}" -out "${outfile}"
  rm -f "${dump}"
}

force_initial_wal_archive() {
  log "forcing initial encrypted WAL archive"
  su-exec postgres psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U postgres -d postgres <<'EOSQL'
SELECT pg_create_restore_point('runtime_boot_' || to_char(clock_timestamp(), 'YYYYMMDDHH24MISS'));
SELECT pg_switch_wal();
EOSQL
  for _ in $(seq 1 30); do
    if ls -1 "${BACKUP_ROOT}"/wal/*.enc >/dev/null 2>&1; then
      log "initial encrypted WAL archive is present"
      return 0
    fi
    sleep 2
  done
  log "initial encrypted WAL archive not observed yet; continuing"
}

backup_loops() {
  (
    while true; do
      sleep "${BASE_BACKUP_INTERVAL_SECONDS}"
      make_base_backup || log "base backup failed"
    done
  ) &
  (
    while true; do
      sleep "${LOGICAL_DUMP_INTERVAL_SECONDS}"
      make_logical_dump || log "logical dump failed"
    done
  ) &
}

sanitize_positive_int() {
  value="$1"
  fallback="$2"
  case "${value}" in
    ''|*[!0-9]*)
      printf '%s\n' "${fallback}"
      ;;
    *)
      if [ "${value}" -le 0 ] 2>/dev/null; then
        printf '%s\n' "${fallback}"
      else
        printf '%s\n' "${value}"
      fi
      ;;
  esac
}

calc_local_dump_keep_count() {
  hours="$(sanitize_positive_int "${LOCAL_DUMP_RETENTION_HOURS:-24}" 24)"
  interval="$(sanitize_positive_int "${LOGICAL_DUMP_INTERVAL_SECONDS:-600}" 600)"
  count=$(( (hours * 3600 + interval - 1) / interval ))
  if [ "${count}" -le 0 ] 2>/dev/null; then
    count=1
  fi
  printf '%s\n' "${count}"
}

prune_keep_latest_n() {
  target_dir="$1"
  pattern="$2"
  keep_count="$(sanitize_positive_int "$3" 1)"
  label="$4"

  [ -d "${target_dir}" ] || return 0

  total_count="$(find "${target_dir}" -maxdepth 1 -type f -name "${pattern}" 2>/dev/null | wc -l | tr -d ' ')"
  total_count="$(sanitize_positive_int "${total_count:-0}" 0)"

  if [ "${total_count}" -le "${keep_count}" ] 2>/dev/null; then
    return 0
  fi

  remove_count=$((total_count - keep_count))
  log "pruning ${label}: total=${total_count} keep=${keep_count} remove=${remove_count}"

  find "${target_dir}" -maxdepth 1 -type f -name "${pattern}" 2>/dev/null \
    | sort \
    | head -n "${remove_count}" \
    | while IFS= read -r old_file; do
        [ -n "${old_file}" ] || continue
        rm -f "${old_file}" && log "deleted ${label} ${old_file}"
      done
}

prune_older_than_hours_keep_newest() {
  target_dir="$1"
  pattern="$2"
  retention_hours="$(sanitize_positive_int "$3" 24)"
  label="$4"

  [ -d "${target_dir}" ] || return 0

  newest_file="$(find "${target_dir}" -maxdepth 1 -type f -name "${pattern}" 2>/dev/null | sort | tail -n 1 || true)"
  [ -n "${newest_file}" ] || return 0

  cutoff_epoch=$(( $(date +%s) - retention_hours * 3600 ))

  find "${target_dir}" -maxdepth 1 -type f -name "${pattern}" 2>/dev/null \
    | sort \
    | while IFS= read -r old_file; do
        [ -n "${old_file}" ] || continue
        [ "${old_file}" = "${newest_file}" ] && continue
        mtime_epoch="$(file_mtime_epoch "${old_file}")"
        if [ "${mtime_epoch}" -lt "${cutoff_epoch}" ] 2>/dev/null; then
          rm -f "${old_file}" && log "deleted ${label} ${old_file}"
        fi
      done
}

run_backup_cleanup_once() {
  if [ "${BACKUP_CLEANUP_ENABLED}" != "true" ]; then
    return 0
  fi

  [ -d "${BACKUP_ROOT}" ] || return 0

  local_dump_keep_count="$(calc_local_dump_keep_count)"
  prune_keep_latest_n "${BACKUP_ROOT}/dumps" 'local-*.dump.enc' "${local_dump_keep_count}" "local dump"
  prune_keep_latest_n "${BACKUP_ROOT}/base" 'base-*.tar.gz.enc' "${BASE_BACKUP_RETAIN_COUNT}" "base backup"
  prune_older_than_hours_keep_newest "${BACKUP_ROOT}/base" 'base-*.tar.gz.enc' "${WAL_RETENTION_HOURS}" "base backup"
  prune_older_than_hours_keep_newest "${BACKUP_ROOT}/wal" '*.enc' "${WAL_RETENTION_HOURS}" "WAL artifact"
}

cleanup_loops() {
  if [ "${BACKUP_CLEANUP_ENABLED}" != "true" ]; then
    log "backup cleanup loop disabled"
    return 0
  fi

  (
    while true; do
      run_backup_cleanup_once || log "backup cleanup failed"
      sleep "${BACKUP_CLEANUP_INTERVAL_SECONDS}"
    done
  ) &
}

setup_local_postgres() {
  [ -n "${LOCAL_DATABASE_PASSWORD}" ] || fail "LOCAL_DATABASE_PASSWORD/DATABASE_PASSWORD is required"
  [ -n "${BACKUP_ENCRYPTION_KEY}" ] || fail "BACKUP_ENCRYPTION_KEY or DATABASE_PASSWORD is required"
  ensure_bucket_layout
  write_pg_helper_scripts

  if [ -s "${LOCAL_PGDATA}/PG_VERSION" ] && [ ! -f "${LOCAL_PGDATA}/.app_ready" ]; then
    log "local PGDATA exists without ready marker; resetting partial cluster"
    rm -rf "${LOCAL_PGDATA}"
  fi

  if [ ! -s "${LOCAL_PGDATA}/PG_VERSION" ]; then
    restored_from_bucket=0
    restored_mode=""
    if [ "${STARTUP_RESTORE_STRATEGY}" = "logical_then_base" ]; then
      if restore_latest_logical_dump; then
        restored_from_bucket=1
        restored_mode="logical"
        log "restored local PostgreSQL from latest logical dump"
      elif restore_latest_base_backup; then
        restored_from_bucket=1
        restored_mode="base"
        log "restored local PostgreSQL from base backup"
      fi
    elif [ "${STARTUP_RESTORE_STRATEGY}" = "base_only" ]; then
      if restore_latest_base_backup; then
        restored_from_bucket=1
        restored_mode="base"
        log "restored local PostgreSQL from base backup"
      fi
    else
      if restore_latest_logical_dump; then
        restored_from_bucket=1
        restored_mode="logical"
        log "restored local PostgreSQL from latest logical dump"
      fi
    fi

    if [ "${restored_from_bucket}" = "0" ]; then
      log "no restore artifacts found; initializing fresh local PostgreSQL"
      rm -rf "${LOCAL_PGDATA}"
      mkdir -p "${LOCAL_PGDATA}"
      chown -R postgres:postgres "${LOCAL_PGDATA}"
      su-exec postgres initdb -D "${LOCAL_PGDATA}" --encoding=UTF8 --locale=C
      configure_pg_for_archive
      start_postgres
      promote_if_in_recovery
      ensure_local_role_db
      make_base_backup
      make_logical_dump || true
      force_initial_wal_archive || true
      backup_loops
      cleanup_loops
      return 0
    fi

    if [ "${restored_mode}" = "logical" ]; then
      force_initial_wal_archive || true
      backup_loops
      cleanup_loops
      return 0
    fi
  fi

  configure_pg_for_archive
  start_postgres
  promote_if_in_recovery
  ensure_local_role_db
  force_initial_wal_archive || true
  backup_loops
  cleanup_loops
}

mkdir -p "${DATA_DIR}" "${DATA_DIR}/redis"

if [ "${LOCAL_POSTGRES_ENABLED}" = "true" ]; then
  setup_local_postgres
fi

if [ -z "${REDIS_HOST:-}" ]; then
  export REDIS_HOST=127.0.0.1
  export REDIS_PORT="${REDIS_PORT:-6379}"
  export REDIS_DB="${REDIS_DB:-0}"
  export REDIS_ENABLE_TLS=false

  if [ -n "${REDIS_PASSWORD:-}" ]; then
    redis-server \
      --bind 127.0.0.1 \
      --port "${REDIS_PORT}" \
      --dir "${DATA_DIR}/redis" \
      --save "" \
      --appendonly no \
      --protected-mode no \
      --maxmemory 128mb \
      --maxmemory-policy allkeys-lru \
      --requirepass "${REDIS_PASSWORD}" &
  else
    redis-server \
      --bind 127.0.0.1 \
      --port "${REDIS_PORT}" \
      --dir "${DATA_DIR}/redis" \
      --save "" \
      --appendonly no \
      --protected-mode no \
      --maxmemory 128mb \
      --maxmemory-policy allkeys-lru &
  fi

  READY=0
  for _ in $(seq 1 50); do
    if [ -n "${REDIS_PASSWORD:-}" ]; then
      if redis-cli -h 127.0.0.1 -p "${REDIS_PORT}" -a "${REDIS_PASSWORD}" ping >/dev/null 2>&1; then
        READY=1
        break
      fi
    else
      if redis-cli -h 127.0.0.1 -p "${REDIS_PORT}" ping >/dev/null 2>&1; then
        READY=1
        break
      fi
    fi
    sleep 0.2
  done

  if [ "${READY}" != "1" ]; then
    fail "local Redis failed to become ready"
  fi
fi

write_runtime_config

if [ "${LOCAL_POSTGRES_ENABLED}" = "true" ]; then
  # The Go config loader uses AutomaticEnv(), so HF DATABASE_* variables override
  # config.yaml. Keep the process environment aligned with the generated local-PG
  # config, otherwise the app can accidentally connect to the source DB.
  export DATABASE_HOST="${LOCAL_DATABASE_HOST}"
  export DATABASE_PORT="${LOCAL_DATABASE_PORT}"
  export DATABASE_USER="${LOCAL_DATABASE_USER}"
  export DATABASE_PASSWORD="${LOCAL_DATABASE_PASSWORD}"
  export DATABASE_DBNAME="${LOCAL_DATABASE_DBNAME}"
  export DATABASE_SSLMODE="disable"
  log "runtime DATABASE_* points to local PostgreSQL ${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DBNAME}"
fi

chown -R app:app "${DATA_DIR}" 2>/dev/null || true


if [ -n "${RESET_ADMIN_PASSWORD_HASH:-}" ]; then
  (
    log "waiting for users table to reset/create admin"
    for _ in $(seq 1 120); do
      if su-exec postgres psql -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U postgres -d "${LOCAL_DATABASE_DBNAME}" -Atqc "SELECT to_regclass('public.users')" | grep -q "users"; then
        su-exec postgres psql -v ON_ERROR_STOP=1 \
          -v admin_email="${ADMIN_EMAIL:-admin@sub2api.local}" \
          -v admin_hash="${RESET_ADMIN_PASSWORD_HASH}" \
          -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U postgres -d "${LOCAL_DATABASE_DBNAME}" <<'EOSQL'
WITH existing_admin AS (
  SELECT id FROM users WHERE role = 'admin' AND deleted_at IS NULL ORDER BY id LIMIT 1
),
updated_admin AS (
  UPDATE users u
  SET email = :'admin_email',
      password_hash = :'admin_hash',
      status = 'active',
      totp_enabled = false,
      totp_secret_encrypted = NULL,
      totp_enabled_at = NULL,
      updated_at = NOW()
  FROM existing_admin ea
  WHERE u.id = ea.id
  RETURNING u.id
),
updated_email_user AS (
  UPDATE users
  SET role = 'admin',
      password_hash = :'admin_hash',
      status = 'active',
      totp_enabled = false,
      totp_secret_encrypted = NULL,
      totp_enabled_at = NULL,
      updated_at = NOW()
  WHERE email = :'admin_email'
    AND deleted_at IS NULL
    AND NOT EXISTS (SELECT 1 FROM updated_admin)
  RETURNING id
)
INSERT INTO users (email, password_hash, role, balance, concurrency, status, created_at, updated_at)
SELECT :'admin_email', :'admin_hash', 'admin', 0, 30, 'active', NOW(), NOW()
WHERE NOT EXISTS (SELECT 1 FROM updated_admin)
  AND NOT EXISTS (SELECT 1 FROM updated_email_user);

SELECT id, email, role, status FROM users WHERE role = 'admin' AND deleted_at IS NULL;
EOSQL
        log "admin password reset/create done"
        exit 0
      fi
      sleep 2
    done
    log "ERROR: users table was not ready for admin reset"
  ) &
fi

exec /app/boot.sh /app/server