Run PostgreSQL locally with bucket backups

start.sh

@@ -1,27 +1,19 @@
 #!/bin/sh
 set -eu
 
-PGDATA=/data/postgresql
-REDIS_DATA_DIR=/data/redis
+PGDATA=/tmp/sub2api-postgresql
+LEGACY_PGDATA=/data/postgresql
+BACKUP_DIR=/data/postgresql-backup
+BACKUP_FILE="$BACKUP_DIR/latest.dump"
+REDIS_DATA_DIR=/tmp/sub2api-redis
 APP_DATA_DIR=/data/sub2api
 APP_PID=""
 REDIS_PID=""
+BACKUP_PID=""
+MIGRATED=0
+NEW_CLUSTER=0
 
-PG_MARKER_DIRS="
-pg_commit_ts
-pg_dynshmem
-pg_multixact/members
-pg_multixact/offsets
-pg_notify
-pg_serial
-pg_snapshots
-pg_stat
-pg_stat_tmp
-pg_subtrans
-pg_twophase
-"
-
-PG_RECREATE_DIRS="
+PG_DIRS="
 pg_commit_ts
 pg_dynshmem
 pg_logical
@@ -45,26 +37,14 @@ pg_wal/summaries
 "
 
 ensure_pg_dirs() {
-  for rel in $PG_RECREATE_DIRS; do
+  for rel in $PG_DIRS; do
     dir="$PGDATA/$rel"
     if [ ! -d "$dir" ]; then
       mkdir -p "$dir"
       chown postgres:postgres "$dir"
       chmod 700 "$dir"
-      echo "Restored PostgreSQL directory: $rel"
     fi
-  done
-
-  for rel in $PG_MARKER_DIRS; do
-    marker="$PGDATA/$rel/.hf-keep"
-    if [ ! -f "$marker" ]; then
-      printf '%s\n' "HF Spaces persistent directory marker" > "$marker"
-      chown postgres:postgres "$marker"
-    fi
-  done
-
-  for rel in pg_logical pg_logical/mappings pg_logical/snapshots pg_multixact pg_replslot pg_tblspc pg_wal pg_wal/archive_status pg_wal/summaries; do
-    rm -f "$PGDATA/$rel/.hf-keep"
+    rm -f "$dir/.hf-keep"
   done
 }
 
@@ -72,18 +52,101 @@ start_postgres() {
   su-exec postgres pg_ctl -D "$PGDATA" -o "-c listen_addresses=127.0.0.1 -c port=5432 -c unix_socket_directories=/tmp" -w start
 }
 
+start_redis() {
+  su-exec redis redis-server \
+    --bind 127.0.0.1 \
+    --port 6379 \
+    --dir "$REDIS_DATA_DIR" \
+    --save 60 1 \
+    --appendonly yes \
+    --appendfsync everysec &
+  REDIS_PID=$!
+}
+
+backup_postgres() {
+  if ! su-exec postgres pg_ctl -D "$PGDATA" status >/dev/null 2>&1; then
+    return 1
+  fi
+
+  mkdir -p "$BACKUP_DIR"
+  local_dump="/tmp/sub2api-backup.$.dump"
+  bucket_dump="$BACKUP_DIR/.latest.$$.dump"
+  rm -f "$local_dump" "$bucket_dump"
+
+  if ! su-exec postgres pg_dump -Fc -d "$DATABASE_DBNAME" -f "$local_dump"; then
+    rm -f "$local_dump"
+    return 1
+  fi
+  chmod 600 "$local_dump"
+
+  if ! cp "$local_dump" "$bucket_dump"; then
+    rm -f "$local_dump" "$bucket_dump"
+    return 1
+  fi
+  if ! mv -f "$bucket_dump" "$BACKUP_FILE"; then
+    rm -f "$local_dump" "$bucket_dump"
+    return 1
+  fi
+
+  rm -f "$local_dump"
+  echo "PostgreSQL backup saved to persistent storage"
+}
+
+cleanup() {
+  trap - EXIT INT TERM
+  if [ -n "$APP_PID" ] && kill -0 "$APP_PID" 2>/dev/null; then
+    kill -TERM "$APP_PID" 2>/dev/null || true
+    wait "$APP_PID" 2>/dev/null || true
+  fi
+  if [ -n "$BACKUP_PID" ] && kill -0 "$BACKUP_PID" 2>/dev/null; then
+    kill -TERM "$BACKUP_PID" 2>/dev/null || true
+    wait "$BACKUP_PID" 2>/dev/null || true
+  fi
+  backup_postgres >/dev/null 2>&1 || true
+  if [ -n "$REDIS_PID" ] && kill -0 "$REDIS_PID" 2>/dev/null; then
+    kill -TERM "$REDIS_PID" 2>/dev/null || true
+    wait "$REDIS_PID" 2>/dev/null || true
+  fi
+  su-exec postgres pg_ctl -D "$PGDATA" -m fast -w stop >/dev/null 2>&1 || true
+}
+trap cleanup EXIT INT TERM
+
 if [ -z "$DATABASE_PASSWORD" ]; then
   echo "DATABASE_PASSWORD is required" >&2
   exit 1
 fi
 
-mkdir -p "$PGDATA" "$REDIS_DATA_DIR" "$APP_DATA_DIR"
-chown -R postgres:postgres "$PGDATA"
-chmod 700 "$PGDATA"
+mkdir -p "$BACKUP_DIR" "$REDIS_DATA_DIR" "$APP_DATA_DIR"
 chown -R redis:redis "$REDIS_DATA_DIR"
 chown -R sub2api:sub2api "$APP_DATA_DIR"
 
+# Prefer the logical backup after the first successful migration. The original
+# bucket-backed cluster remains untouched as a one-time recovery source.
+if [ ! -s "$PGDATA/PG_VERSION" ] && [ ! -s "$BACKUP_FILE" ] && [ -s "$LEGACY_PGDATA/PG_VERSION" ]; then
+  echo "Migrating legacy PostgreSQL data to local runtime storage"
+  copy_attempt=0
+  while :; do
+    rm -rf "$PGDATA"
+    mkdir -p "$PGDATA"
+    if cp -a "$LEGACY_PGDATA/." "$PGDATA/"; then
+      break
+    fi
+    copy_attempt=$((copy_attempt + 1))
+    if [ "$copy_attempt" -ge 3 ]; then
+      echo "Legacy PostgreSQL data copy failed" >&2
+      exit 1
+    fi
+    sleep 5
+  done
+  rm -f "$PGDATA/postmaster.pid" "$PGDATA/backup_label" "$PGDATA/tablespace_map"
+  MIGRATED=1
+fi
+
 if [ ! -s "$PGDATA/PG_VERSION" ]; then
+  NEW_CLUSTER=1
+  mkdir -p "$PGDATA"
+  chown postgres:postgres "$PGDATA"
+  chmod 700 "$PGDATA"
   PASSWORD_FILE=$(mktemp)
   chmod 600 "$PASSWORD_FILE"
   printf '%s\n' "$DATABASE_PASSWORD" > "$PASSWORD_FILE"
@@ -92,37 +155,34 @@ if [ ! -s "$PGDATA/PG_VERSION" ]; then
   rm -f "$PASSWORD_FILE"
 fi
 
-rm -f "$PGDATA/postmaster.pid"
 ensure_pg_dirs
+chown -R postgres:postgres "$PGDATA"
+chmod 700 "$PGDATA"
+rm -f "$PGDATA/postmaster.pid"
 start_postgres
 
 if ! su-exec postgres psql -h /tmp -U "$DATABASE_USER" -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = 'sub2api'" | grep -q 1; then
   su-exec postgres createdb -h /tmp -U "$DATABASE_USER" -O "$DATABASE_USER" "$DATABASE_DBNAME"
 fi
 
-su-exec redis redis-server \
-  --bind 127.0.0.1 \
-  --port 6379 \
-  --dir "$REDIS_DATA_DIR" \
-  --save 60 1 \
-  --appendonly yes \
-  --appendfsync everysec &
-REDIS_PID=$!
-
-cleanup() {
-  trap - EXIT INT TERM
-  if [ -n "$APP_PID" ] && kill -0 "$APP_PID" 2>/dev/null; then
-    kill -TERM "$APP_PID" 2>/dev/null || true
-    wait "$APP_PID" 2>/dev/null || true
+if [ "$NEW_CLUSTER" -eq 1 ] && [ "$MIGRATED" -eq 0 ] && [ -s "$BACKUP_FILE" ]; then
+  echo "Restoring PostgreSQL from persistent logical backup"
+  restore_dump=/tmp/sub2api-restore.dump
+  rm -f "$restore_dump"
+  if ! cp "$BACKUP_FILE" "$restore_dump"; then
+    echo "Persistent PostgreSQL backup could not be read" >&2
+    exit 1
   fi
-  if [ -n "$REDIS_PID" ] && kill -0 "$REDIS_PID" 2>/dev/null; then
-    kill -TERM "$REDIS_PID" 2>/dev/null || true
-    wait "$REDIS_PID" 2>/dev/null || true
+  if ! su-exec postgres pg_restore -l "$restore_dump" >/dev/null 2>&1; then
+    echo "Persistent PostgreSQL backup is invalid" >&2
+    exit 1
   fi
-  su-exec postgres pg_ctl -D "$PGDATA" -m fast -w stop >/dev/null 2>&1 || true
-}
-trap cleanup EXIT INT TERM
+  su-exec postgres pg_restore --clean --if-exists --no-owner --exit-on-error -d "$DATABASE_DBNAME" "$restore_dump"
+  rm -f "$restore_dump"
+  echo "PostgreSQL restore completed"
+fi
 
+start_redis
 attempt=0
 until redis-cli -h 127.0.0.1 -p 6379 ping 2>/dev/null | grep -q PONG; do
   if ! kill -0 "$REDIS_PID" 2>/dev/null; then
@@ -140,10 +200,18 @@ done
 su-exec sub2api /app/sub2api &
 APP_PID=$!
 
+(
+  sleep 15
+  while kill -0 "$APP_PID" 2>/dev/null; do
+    backup_postgres || echo "PostgreSQL periodic backup failed" >&2
+    sleep 30
+  done
+) &
+BACKUP_PID=$!
+
 while kill -0 "$APP_PID" 2>/dev/null; do
-  ensure_pg_dirs
   if ! su-exec postgres pg_ctl -D "$PGDATA" status >/dev/null 2>&1; then
-    echo "PostgreSQL stopped unexpectedly; attempting recovery" >&2
+    echo "PostgreSQL stopped unexpectedly; attempting local recovery" >&2
     rm -f "$PGDATA/postmaster.pid"
     if ! start_postgres; then
       echo "PostgreSQL recovery failed; stopping Sub2API" >&2
@@ -151,7 +219,6 @@ while kill -0 "$APP_PID" 2>/dev/null; do
       wait "$APP_PID" 2>/dev/null || true
       exit 1
     fi
-    echo "PostgreSQL recovered"
   fi
   sleep 2
 done