Spaces:
Running
Running
| set -eu | |
| umask 077 | |
| log() { | |
| printf '[runtime-hf] %s\n' "$*" >&2 | |
| } | |
| fail() { | |
| log "ERROR: $*" | |
| exit 1 | |
| } | |
| yaml_escape() { | |
| printf "%s" "$1" | sed "s/'/''/g" | |
| } | |
| sql_escape() { | |
| printf "%s" "$1" | sed "s/'/''/g" | |
| } | |
| if [ -z "${DATA_DIR:-}" ] && [ -d /data ]; then | |
| export DATA_DIR=/data/appdata | |
| else | |
| export DATA_DIR="${DATA_DIR:-/app/data}" | |
| fi | |
| export AUTO_SETUP="${AUTO_SETUP:-true}" | |
| export SERVER_HOST=0.0.0.0 | |
| export SERVER_PORT="${SERVER_PORT:-8080}" | |
| export SERVER_MODE="${SERVER_MODE:-release}" | |
| export RUN_MODE="${RUN_MODE:-standard}" | |
| export TZ="${TZ:-Asia/Shanghai}" | |
| # Production mode: run PostgreSQL inside this Space on ephemeral disk and | |
| # archive encrypted WAL/base/logical backups to the mounted HF Storage Bucket. | |
| export LOCAL_POSTGRES_ENABLED="${LOCAL_POSTGRES_ENABLED:-true}" | |
| if [ -z "${BACKUP_ROOT:-}" ]; then | |
| export BACKUP_ROOT="/data/sub2api-pg-backups" | |
| fi | |
| export LOCAL_PGDATA="${LOCAL_PGDATA:-/tmp/app-pgdata}" | |
| export LOCAL_DATABASE_HOST="127.0.0.1" | |
| export LOCAL_DATABASE_PORT="${LOCAL_DATABASE_PORT:-5432}" | |
| export LOCAL_DATABASE_USER="${LOCAL_DATABASE_USER:-appuser}" | |
| export LOCAL_DATABASE_DBNAME="${LOCAL_DATABASE_DBNAME:-appdb}" | |
| export LOCAL_DATABASE_PASSWORD="${LOCAL_DATABASE_PASSWORD:-${DATABASE_PASSWORD:-}}" | |
| export BACKUP_ENCRYPTION_KEY="${BACKUP_ENCRYPTION_KEY:-${DATABASE_PASSWORD:-}}" | |
| export LOCAL_PG_ARCHIVE_TIMEOUT="${LOCAL_PG_ARCHIVE_TIMEOUT:-30s}" | |
| export LOCAL_PG_PROMOTE_WAIT_SECONDS="${LOCAL_PG_PROMOTE_WAIT_SECONDS:-900}" | |
| export BASE_BACKUP_INTERVAL_SECONDS="${BASE_BACKUP_INTERVAL_SECONDS:-21600}" | |
| export LOGICAL_DUMP_INTERVAL_SECONDS="${LOGICAL_DUMP_INTERVAL_SECONDS:-600}" | |
| export BACKUP_CLEANUP_ENABLED="${BACKUP_CLEANUP_ENABLED:-true}" | |
| export BACKUP_CLEANUP_INTERVAL_SECONDS="${BACKUP_CLEANUP_INTERVAL_SECONDS:-3600}" | |
| export LOCAL_DUMP_RETENTION_HOURS="${LOCAL_DUMP_RETENTION_HOURS:-24}" | |
| export BASE_BACKUP_RETAIN_COUNT="${BASE_BACKUP_RETAIN_COUNT:-4}" | |
| export WAL_RETENTION_HOURS="${WAL_RETENTION_HOURS:-4}" | |
| export STARTUP_RESTORE_STRATEGY="${STARTUP_RESTORE_STRATEGY:-logical_then_base}" | |
| file_mtime_epoch() { | |
| stat -c %Y "$1" 2>/dev/null || stat -f %m "$1" 2>/dev/null || printf '0\n' | |
| } | |
| # Conservative DB pool profile for HF cpu-basic: keep app-side pool below | |
| # PostgreSQL max_connections, leaving headroom for maintenance, backups and psql. | |
| export DATABASE_MAX_OPEN_CONNS="${DATABASE_MAX_OPEN_CONNS:-160}" | |
| export DATABASE_MAX_IDLE_CONNS="${DATABASE_MAX_IDLE_CONNS:-60}" | |
| export DATABASE_CONN_MAX_LIFETIME_MINUTES="${DATABASE_CONN_MAX_LIFETIME_MINUTES:-30}" | |
| export DATABASE_CONN_MAX_IDLE_TIME_MINUTES="${DATABASE_CONN_MAX_IDLE_TIME_MINUTES:-5}" | |
| write_runtime_config() { | |
| config_file="${DATA_DIR}/config.yaml" | |
| install_lock="${DATA_DIR}/.installed" | |
| user_concurrency=5 | |
| if [ "${RUN_MODE}" = "simple" ]; then | |
| user_concurrency=30 | |
| fi | |
| if [ -z "${JWT_SECRET:-}" ]; then | |
| fail "JWT_SECRET is required when bootstrapping from env on Hugging Face" | |
| fi | |
| db_host="${DATABASE_HOST}" | |
| db_port="${DATABASE_PORT}" | |
| db_user="${DATABASE_USER}" | |
| db_password="${DATABASE_PASSWORD}" | |
| db_name="${DATABASE_DBNAME}" | |
| db_sslmode="${DATABASE_SSLMODE:-require}" | |
| if [ "${LOCAL_POSTGRES_ENABLED}" = "true" ]; then | |
| db_host="${LOCAL_DATABASE_HOST}" | |
| db_port="${LOCAL_DATABASE_PORT}" | |
| db_user="${LOCAL_DATABASE_USER}" | |
| db_password="${LOCAL_DATABASE_PASSWORD}" | |
| db_name="${LOCAL_DATABASE_DBNAME}" | |
| db_sslmode="disable" | |
| fi | |
| cat > "${config_file}" <<EOCFG | |
| server: | |
| host: '$(yaml_escape "${SERVER_HOST}")' | |
| port: ${SERVER_PORT} | |
| mode: '$(yaml_escape "${SERVER_MODE}")' | |
| database: | |
| host: '$(yaml_escape "${db_host}")' | |
| port: ${db_port} | |
| user: '$(yaml_escape "${db_user}")' | |
| password: '$(yaml_escape "${db_password}")' | |
| dbname: '$(yaml_escape "${db_name}")' | |
| sslmode: '$(yaml_escape "${db_sslmode}")' | |
| max_open_conns: ${DATABASE_MAX_OPEN_CONNS} | |
| max_idle_conns: ${DATABASE_MAX_IDLE_CONNS} | |
| conn_max_lifetime_minutes: ${DATABASE_CONN_MAX_LIFETIME_MINUTES} | |
| conn_max_idle_time_minutes: ${DATABASE_CONN_MAX_IDLE_TIME_MINUTES} | |
| redis: | |
| host: '$(yaml_escape "${REDIS_HOST}")' | |
| port: ${REDIS_PORT} | |
| password: '$(yaml_escape "${REDIS_PASSWORD:-}")' | |
| db: ${REDIS_DB} | |
| enable_tls: ${REDIS_ENABLE_TLS} | |
| jwt: | |
| secret: '$(yaml_escape "${JWT_SECRET}")' | |
| expire_hour: ${JWT_EXPIRE_HOUR:-24} | |
| default: | |
| user_concurrency: ${user_concurrency} | |
| user_balance: 0 | |
| api_key_prefix: sk- | |
| rate_multiplier: 1 | |
| rate_limit: | |
| requests_per_minute: 60 | |
| burst_size: 10 | |
| timezone: '$(yaml_escape "${TZ}")' | |
| EOCFG | |
| chmod 600 "${config_file}" | |
| printf 'installed_at=%s\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" > "${install_lock}" | |
| chmod 400 "${install_lock}" | |
| } | |
| ensure_bucket_layout() { | |
| [ -d /data ] || fail "HF Storage Bucket is not mounted at /data" | |
| [ -w /data ] || fail "HF Storage Bucket mount /data is not writable" | |
| mkdir -p \ | |
| "${BACKUP_ROOT}/base" \ | |
| "${BACKUP_ROOT}/wal" \ | |
| "${BACKUP_ROOT}/dumps" \ | |
| "${BACKUP_ROOT}/state" \ | |
| "${BACKUP_ROOT}/tmp" | |
| # archive_command runs as the postgres user, so the mounted bucket subtree | |
| # must be writable by postgres. HF bucket mounts support chown in this Space. | |
| chown -R postgres:postgres "${BACKUP_ROOT}" 2>/dev/null || true | |
| } | |
| write_pg_helper_scripts() { | |
| cat > /app/archive_wal.sh <<'EOS' | |
| #!/bin/sh | |
| set -eu | |
| src="$1" | |
| name="$2" | |
| root="${BACKUP_ROOT:?BACKUP_ROOT required}" | |
| : "${BACKUP_ENCRYPTION_KEY:?BACKUP_ENCRYPTION_KEY required}" | |
| mkdir -p "${root}/wal" "${root}/tmp" | |
| out="${root}/wal/${name}.enc" | |
| [ -s "${out}" ] && exit 0 | |
| tmp="${root}/tmp/${name}.enc.$$" | |
| openssl enc -aes-256-cbc -salt -pbkdf2 -pass env:BACKUP_ENCRYPTION_KEY -in "${src}" -out "${tmp}" | |
| mv "${tmp}" "${out}" | |
| EOS | |
| cat > /app/restore_wal.sh <<'EOS' | |
| #!/bin/sh | |
| set -eu | |
| name="$1" | |
| dest="$2" | |
| root="${BACKUP_ROOT:?BACKUP_ROOT required}" | |
| : "${BACKUP_ENCRYPTION_KEY:?BACKUP_ENCRYPTION_KEY required}" | |
| infile="${root}/wal/${name}.enc" | |
| [ -s "${infile}" ] || exit 1 | |
| openssl enc -d -aes-256-cbc -pbkdf2 -pass env:BACKUP_ENCRYPTION_KEY -in "${infile}" -out "${dest}" | |
| EOS | |
| chmod 755 /app/archive_wal.sh /app/restore_wal.sh | |
| } | |
| configure_pg_for_archive() { | |
| conf="${LOCAL_PGDATA}/postgresql.conf" | |
| hba="${LOCAL_PGDATA}/pg_hba.conf" | |
| marker="# runtime-hf-local-pg" | |
| if ! grep -q "${marker}" "${conf}" 2>/dev/null; then | |
| cat >> "${conf}" <<EOCONF | |
| ${marker} | |
| listen_addresses = '127.0.0.1' | |
| port = ${LOCAL_DATABASE_PORT} | |
| unix_socket_directories = '/tmp' | |
| wal_level = replica | |
| archive_mode = on | |
| archive_timeout = '${LOCAL_PG_ARCHIVE_TIMEOUT}' | |
| archive_command = '/app/archive_wal.sh "%p" "%f"' | |
| max_wal_size = '1GB' | |
| checkpoint_timeout = '5min' | |
| EOCONF | |
| fi | |
| tuning_marker="# runtime-hf-pg-tuning-v2" | |
| if ! grep -q "${tuning_marker}" "${conf}" 2>/dev/null; then | |
| cat >> "${conf}" <<EOCONF | |
| ${tuning_marker} | |
| max_connections = 200 | |
| shared_buffers = '1GB' | |
| effective_cache_size = '6GB' | |
| work_mem = '8MB' | |
| maintenance_work_mem = '256MB' | |
| max_wal_size = '2GB' | |
| checkpoint_timeout = '10min' | |
| wal_compression = on | |
| EOCONF | |
| fi | |
| if ! grep -q "runtime-hf-local-pg" "${hba}" 2>/dev/null; then | |
| cat >> "${hba}" <<EOHBA | |
| # runtime-hf-local-pg | |
| local all all trust | |
| host all all 127.0.0.1/32 md5 | |
| host replication all 127.0.0.1/32 trust | |
| EOHBA | |
| fi | |
| } | |
| restore_latest_base_backup() { | |
| latest="$(ls -1 "${BACKUP_ROOT}"/base/base-*.tar.gz.enc 2>/dev/null | sort | tail -1 || true)" | |
| [ -n "${latest}" ] || return 1 | |
| log "restoring local PostgreSQL from encrypted base backup ${latest}" | |
| rm -rf "${LOCAL_PGDATA}" | |
| mkdir -p "${LOCAL_PGDATA}" | |
| tmp="/tmp/app-base-restore.tar.gz" | |
| openssl enc -d -aes-256-cbc -pbkdf2 -pass env:BACKUP_ENCRYPTION_KEY -in "${latest}" -out "${tmp}" | |
| tar -xzf "${tmp}" -C "${LOCAL_PGDATA}" | |
| rm -f "${tmp}" "${LOCAL_PGDATA}/postmaster.pid" | |
| configure_pg_for_archive | |
| cat >> "${LOCAL_PGDATA}/postgresql.conf" <<'EOCONF' | |
| restore_command = '/app/restore_wal.sh "%f" "%p"' | |
| EOCONF | |
| touch "${LOCAL_PGDATA}/recovery.signal" | |
| touch "${LOCAL_PGDATA}/.app_ready" | |
| chown postgres:postgres "${LOCAL_PGDATA}/.app_ready" | |
| chown -R postgres:postgres "${LOCAL_PGDATA}" | |
| return 0 | |
| } | |
| start_postgres() { | |
| log "starting local PostgreSQL on 127.0.0.1:${LOCAL_DATABASE_PORT}" | |
| chown -R postgres:postgres "${LOCAL_PGDATA}" | |
| rm -f /tmp/app-postgres.log | |
| if ! su-exec postgres pg_ctl -D "${LOCAL_PGDATA}" -l /tmp/app-postgres.log -w start; then | |
| log "PostgreSQL failed to start; log follows" | |
| cat /tmp/app-postgres.log 2>/dev/null || true | |
| return 1 | |
| fi | |
| } | |
| promote_if_in_recovery() { | |
| if [ "$(su-exec postgres psql -At -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U postgres -d postgres -c 'SELECT pg_is_in_recovery();')" = "t" ]; then | |
| log "promoting restored PostgreSQL out of recovery" | |
| su-exec postgres pg_ctl -D "${LOCAL_PGDATA}" promote || log "pg_ctl promote was not needed or not supported; waiting for recovery to finish" | |
| for _ in $(seq 1 "${LOCAL_PG_PROMOTE_WAIT_SECONDS}"); do | |
| if [ "$(su-exec postgres psql -At -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U postgres -d postgres -c 'SELECT pg_is_in_recovery();')" = "f" ]; then | |
| log "restored PostgreSQL is writable" | |
| return 0 | |
| fi | |
| sleep 1 | |
| done | |
| fail "restored PostgreSQL is still in recovery after waiting" | |
| fi | |
| } | |
| ensure_local_role_db() { | |
| su-exec postgres psql -v ON_ERROR_STOP=1 -v role_password="${LOCAL_DATABASE_PASSWORD}" -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U postgres -d postgres <<EOSQL | |
| SELECT 'CREATE ROLE ${LOCAL_DATABASE_USER} LOGIN SUPERUSER' | |
| WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${LOCAL_DATABASE_USER}')\gexec | |
| ALTER ROLE ${LOCAL_DATABASE_USER} WITH LOGIN SUPERUSER PASSWORD :'role_password'; | |
| SELECT 'CREATE DATABASE ${LOCAL_DATABASE_DBNAME} OWNER ${LOCAL_DATABASE_USER}' | |
| WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '${LOCAL_DATABASE_DBNAME}')\gexec | |
| EOSQL | |
| } | |
| restore_latest_logical_dump() { | |
| latest="$(ls -1 "${BACKUP_ROOT}"/dumps/local-*.dump.enc 2>/dev/null | sort | tail -1 || true)" | |
| [ -n "${latest}" ] || return 1 | |
| log "restoring local PostgreSQL from encrypted logical dump ${latest}" | |
| rm -rf "${LOCAL_PGDATA}" | |
| mkdir -p "${LOCAL_PGDATA}" | |
| chown -R postgres:postgres "${LOCAL_PGDATA}" | |
| su-exec postgres initdb -D "${LOCAL_PGDATA}" --encoding=UTF8 --locale=C | |
| configure_pg_for_archive | |
| start_postgres | |
| promote_if_in_recovery | |
| ensure_local_role_db | |
| dump="/tmp/runtime-latest-local.dump" | |
| rm -f "${dump}" | |
| openssl enc -d -aes-256-cbc -pbkdf2 -pass env:BACKUP_ENCRYPTION_KEY -in "${latest}" -out "${dump}" | |
| su-exec postgres psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U postgres -d postgres <<EOSQL | |
| SELECT pg_terminate_backend(pid) | |
| FROM pg_stat_activity | |
| WHERE datname = '${LOCAL_DATABASE_DBNAME}' | |
| AND pid <> pg_backend_pid(); | |
| DROP DATABASE IF EXISTS ${LOCAL_DATABASE_DBNAME} WITH (FORCE); | |
| CREATE DATABASE ${LOCAL_DATABASE_DBNAME} OWNER ${LOCAL_DATABASE_USER}; | |
| EOSQL | |
| PGPASSWORD="${LOCAL_DATABASE_PASSWORD}" \ | |
| pg_restore -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U "${LOCAL_DATABASE_USER}" -d "${LOCAL_DATABASE_DBNAME}" \ | |
| --no-owner --no-privileges --exit-on-error "${dump}" | |
| rm -f "${dump}" | |
| touch "${LOCAL_PGDATA}/.app_ready" | |
| chown postgres:postgres "${LOCAL_PGDATA}/.app_ready" | |
| return 0 | |
| } | |
| make_base_backup() { | |
| ts="$(date -u +%Y%m%dT%H%M%SZ)" | |
| tmpdir="/tmp/base-${ts}" | |
| tarball="/tmp/base-${ts}.tar.gz" | |
| outfile="${BACKUP_ROOT}/base/base-${ts}.tar.gz.enc" | |
| rm -rf "${tmpdir}" "${tarball}" | |
| mkdir -p "${tmpdir}" | |
| chown postgres:postgres "${tmpdir}" | |
| log "creating encrypted physical base backup ${outfile}" | |
| su-exec postgres pg_basebackup -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U postgres -D "${tmpdir}" -Fp -Xs | |
| tar -C "${tmpdir}" -czf "${tarball}" . | |
| openssl enc -aes-256-cbc -salt -pbkdf2 -pass env:BACKUP_ENCRYPTION_KEY -in "${tarball}" -out "${outfile}" | |
| rm -rf "${tmpdir}" "${tarball}" | |
| printf '%s\n' "${outfile}" > "${BACKUP_ROOT}/state/latest-base.txt" | |
| } | |
| make_logical_dump() { | |
| ts="$(date -u +%Y%m%dT%H%M%SZ)" | |
| dump="/tmp/local-${ts}.dump" | |
| outfile="${BACKUP_ROOT}/dumps/local-${ts}.dump.enc" | |
| log "creating encrypted logical dump ${outfile}" | |
| PGPASSWORD="${LOCAL_DATABASE_PASSWORD}" \ | |
| pg_dump -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U "${LOCAL_DATABASE_USER}" -d "${LOCAL_DATABASE_DBNAME}" \ | |
| -Fc --no-owner --no-privileges -f "${dump}" | |
| openssl enc -aes-256-cbc -salt -pbkdf2 -pass env:BACKUP_ENCRYPTION_KEY -in "${dump}" -out "${outfile}" | |
| rm -f "${dump}" | |
| } | |
| force_initial_wal_archive() { | |
| log "forcing initial encrypted WAL archive" | |
| su-exec postgres psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -p "${LOCAL_DATABASE_PORT}" -U postgres -d postgres <<'EOSQL' | |
| SELECT pg_create_restore_point('runtime_boot_' || to_char(clock_timestamp(), 'YYYYMMDDHH24MISS')); | |
| SELECT pg_switch_wal(); | |
| EOSQL | |
| for _ in $(seq 1 30); do | |
| if ls -1 "${BACKUP_ROOT}"/wal/*.enc >/dev/null 2>&1; then | |
| log "initial encrypted WAL archive is present" | |
| return 0 | |
| fi | |
| sleep 2 | |
| done | |
| log "initial encrypted WAL archive not observed yet; continuing" | |
| } | |
| backup_loops() { | |
| ( | |
| while true; do | |
| sleep "${BASE_BACKUP_INTERVAL_SECONDS}" | |
| make_base_backup || log "base backup failed" | |
| done | |
| ) & | |
| ( | |
| while true; do | |
| sleep "${LOGICAL_DUMP_INTERVAL_SECONDS}" | |
| make_logical_dump || log "logical dump failed" | |
| done | |
| ) & | |
| } | |
| sanitize_positive_int() { | |
| value="$1" | |
| fallback="$2" | |
| case "${value}" in | |
| ''|*[!0-9]*) | |
| printf '%s\n' "${fallback}" | |
| ;; | |
| *) | |
| if [ "${value}" -le 0 ] 2>/dev/null; then | |
| printf '%s\n' "${fallback}" | |
| else | |
| printf '%s\n' "${value}" | |
| fi | |
| ;; | |
| esac | |
| } | |
| calc_local_dump_keep_count() { | |
| hours="$(sanitize_positive_int "${LOCAL_DUMP_RETENTION_HOURS:-24}" 24)" | |
| interval="$(sanitize_positive_int "${LOGICAL_DUMP_INTERVAL_SECONDS:-600}" 600)" | |
| count=$(( (hours * 3600 + interval - 1) / interval )) | |
| if [ "${count}" -le 0 ] 2>/dev/null; then | |
| count=1 | |
| fi | |
| printf '%s\n' "${count}" | |
| } | |
| prune_keep_latest_n() { | |
| target_dir="$1" | |
| pattern="$2" | |
| keep_count="$(sanitize_positive_int "$3" 1)" | |
| label="$4" | |
| [ -d "${target_dir}" ] || return 0 | |
| total_count="$(find "${target_dir}" -maxdepth 1 -type f -name "${pattern}" 2>/dev/null | wc -l | tr -d ' ')" | |
| total_count="$(sanitize_positive_int "${total_count:-0}" 0)" | |
| if [ "${total_count}" -le "${keep_count}" ] 2>/dev/null; then | |
| return 0 | |
| fi | |
| remove_count=$((total_count - keep_count)) | |
| log "pruning ${label}: total=${total_count} keep=${keep_count} remove=${remove_count}" | |
| find "${target_dir}" -maxdepth 1 -type f -name "${pattern}" 2>/dev/null \ | |
| | sort \ | |
| | head -n "${remove_count}" \ | |
| | while IFS= read -r old_file; do | |
| [ -n "${old_file}" ] || continue | |
| rm -f "${old_file}" && log "deleted ${label} ${old_file}" | |
| done | |
| } | |
| prune_older_than_hours_keep_newest() { | |
| target_dir="$1" | |
| pattern="$2" | |
| retention_hours="$(sanitize_positive_int "$3" 24)" | |
| label="$4" | |
| [ -d "${target_dir}" ] || return 0 | |
| newest_file="$(find "${target_dir}" -maxdepth 1 -type f -name "${pattern}" 2>/dev/null | sort | tail -n 1 || true)" | |
| [ -n "${newest_file}" ] || return 0 | |
| cutoff_epoch=$(( $(date +%s) - retention_hours * 3600 )) | |
| find "${target_dir}" -maxdepth 1 -type f -name "${pattern}" 2>/dev/null \ | |
| | sort \ | |
| | while IFS= read -r old_file; do | |
| [ -n "${old_file}" ] || continue | |
| [ "${old_file}" = "${newest_file}" ] && continue | |
| mtime_epoch="$(file_mtime_epoch "${old_file}")" | |
| if [ "${mtime_epoch}" -lt "${cutoff_epoch}" ] 2>/dev/null; then | |
| rm -f "${old_file}" && log "deleted ${label} ${old_file}" | |
| fi | |
| done | |
| } | |
| run_backup_cleanup_once() { | |
| if [ "${BACKUP_CLEANUP_ENABLED}" != "true" ]; then | |
| return 0 | |
| fi | |
| [ -d "${BACKUP_ROOT}" ] || return 0 | |
| local_dump_keep_count="$(calc_local_dump_keep_count)" | |
| prune_keep_latest_n "${BACKUP_ROOT}/dumps" 'local-*.dump.enc' "${local_dump_keep_count}" "local dump" | |
| prune_keep_latest_n "${BACKUP_ROOT}/base" 'base-*.tar.gz.enc' "${BASE_BACKUP_RETAIN_COUNT}" "base backup" | |
| prune_older_than_hours_keep_newest "${BACKUP_ROOT}/base" 'base-*.tar.gz.enc' "${WAL_RETENTION_HOURS}" "base backup" | |
| prune_older_than_hours_keep_newest "${BACKUP_ROOT}/wal" '*.enc' "${WAL_RETENTION_HOURS}" "WAL artifact" | |
| } | |
| cleanup_loops() { | |
| if [ "${BACKUP_CLEANUP_ENABLED}" != "true" ]; then | |
| log "backup cleanup loop disabled" | |
| return 0 | |
| fi | |
| ( | |
| while true; do | |
| run_backup_cleanup_once || log "backup cleanup failed" | |
| sleep "${BACKUP_CLEANUP_INTERVAL_SECONDS}" | |
| done | |
| ) & | |
| } | |
| setup_local_postgres() { | |
| [ -n "${LOCAL_DATABASE_PASSWORD}" ] || fail "LOCAL_DATABASE_PASSWORD/DATABASE_PASSWORD is required" | |
| [ -n "${BACKUP_ENCRYPTION_KEY}" ] || fail "BACKUP_ENCRYPTION_KEY or DATABASE_PASSWORD is required" | |
| ensure_bucket_layout | |
| write_pg_helper_scripts | |
| if [ -s "${LOCAL_PGDATA}/PG_VERSION" ] && [ ! -f "${LOCAL_PGDATA}/.app_ready" ]; then | |
| log "local PGDATA exists without ready marker; resetting partial cluster" | |
| rm -rf "${LOCAL_PGDATA}" | |
| fi | |
| if [ ! -s "${LOCAL_PGDATA}/PG_VERSION" ]; then | |
| restored_from_bucket=0 | |
| restored_mode="" | |
| if [ "${STARTUP_RESTORE_STRATEGY}" = "logical_then_base" ]; then | |
| if restore_latest_logical_dump; then | |
| restored_from_bucket=1 | |
| restored_mode="logical" | |
| log "restored local PostgreSQL from latest logical dump" | |
| elif restore_latest_base_backup; then | |
| restored_from_bucket=1 | |
| restored_mode="base" | |
| log "restored local PostgreSQL from base backup" | |
| fi | |
| elif [ "${STARTUP_RESTORE_STRATEGY}" = "base_only" ]; then | |
| if restore_latest_base_backup; then | |
| restored_from_bucket=1 | |
| restored_mode="base" | |
| log "restored local PostgreSQL from base backup" | |
| fi | |
| else | |
| if restore_latest_logical_dump; then | |
| restored_from_bucket=1 | |
| restored_mode="logical" | |
| log "restored local PostgreSQL from latest logical dump" | |
| fi | |
| fi | |
| if [ "${restored_from_bucket}" = "0" ]; then | |
| log "no restore artifacts found; initializing fresh local PostgreSQL" | |
| rm -rf "${LOCAL_PGDATA}" | |
| mkdir -p "${LOCAL_PGDATA}" | |
| chown -R postgres:postgres "${LOCAL_PGDATA}" | |
| su-exec postgres initdb -D "${LOCAL_PGDATA}" --encoding=UTF8 --locale=C | |
| configure_pg_for_archive | |
| start_postgres | |
| promote_if_in_recovery | |
| ensure_local_role_db | |
| make_base_backup | |
| make_logical_dump || true | |
| force_initial_wal_archive || true | |
| backup_loops | |
| cleanup_loops | |
| return 0 | |
| fi | |
| if [ "${restored_mode}" = "logical" ]; then | |
| force_initial_wal_archive || true | |
| backup_loops | |
| cleanup_loops | |
| return 0 | |
| fi | |
| fi | |
| configure_pg_for_archive | |
| start_postgres | |
| promote_if_in_recovery | |
| ensure_local_role_db | |
| force_initial_wal_archive || true | |
| backup_loops | |
| cleanup_loops | |
| } | |
| mkdir -p "${DATA_DIR}" "${DATA_DIR}/redis" | |
| if [ "${LOCAL_POSTGRES_ENABLED}" = "true" ]; then | |
| setup_local_postgres | |
| fi | |
| if [ -z "${REDIS_HOST:-}" ]; then | |
| export REDIS_HOST=127.0.0.1 | |
| export REDIS_PORT="${REDIS_PORT:-6379}" | |
| export REDIS_DB="${REDIS_DB:-0}" | |
| export REDIS_ENABLE_TLS=false | |
| if [ -n "${REDIS_PASSWORD:-}" ]; then | |
| redis-server \ | |
| --bind 127.0.0.1 \ | |
| --port "${REDIS_PORT}" \ | |
| --dir "${DATA_DIR}/redis" \ | |
| --save "" \ | |
| --appendonly no \ | |
| --protected-mode no \ | |
| --maxmemory 128mb \ | |
| --maxmemory-policy allkeys-lru \ | |
| --requirepass "${REDIS_PASSWORD}" & | |
| else | |
| redis-server \ | |
| --bind 127.0.0.1 \ | |
| --port "${REDIS_PORT}" \ | |
| --dir "${DATA_DIR}/redis" \ | |
| --save "" \ | |
| --appendonly no \ | |
| --protected-mode no \ | |
| --maxmemory 128mb \ | |
| --maxmemory-policy allkeys-lru & | |
| fi | |
| READY=0 | |
| for _ in $(seq 1 50); do | |
| if [ -n "${REDIS_PASSWORD:-}" ]; then | |
| if redis-cli -h 127.0.0.1 -p "${REDIS_PORT}" -a "${REDIS_PASSWORD}" ping >/dev/null 2>&1; then | |
| READY=1 | |
| break | |
| fi | |
| else | |
| if redis-cli -h 127.0.0.1 -p "${REDIS_PORT}" ping >/dev/null 2>&1; then | |
| READY=1 | |
| break | |
| fi | |
| fi | |
| sleep 0.2 | |
| done | |
| if [ "${READY}" != "1" ]; then | |
| fail "local Redis failed to become ready" | |
| fi | |
| fi | |
| write_runtime_config | |
| if [ "${LOCAL_POSTGRES_ENABLED}" = "true" ]; then | |
| # The Go config loader uses AutomaticEnv(), so HF DATABASE_* variables override | |
| # config.yaml. Keep the process environment aligned with the generated local-PG | |
| # config, otherwise the app can accidentally connect to the source DB. | |
| export DATABASE_HOST="${LOCAL_DATABASE_HOST}" | |
| export DATABASE_PORT="${LOCAL_DATABASE_PORT}" | |
| export DATABASE_USER="${LOCAL_DATABASE_USER}" | |
| export DATABASE_PASSWORD="${LOCAL_DATABASE_PASSWORD}" | |
| export DATABASE_DBNAME="${LOCAL_DATABASE_DBNAME}" | |
| export DATABASE_SSLMODE="disable" | |
| log "runtime DATABASE_* points to local PostgreSQL ${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DBNAME}" | |
| fi | |
| chown -R app:app "${DATA_DIR}" 2>/dev/null || true | |
| exec /app/boot.sh /app/server | |