syncs atomic .backup sqlite db to S3

entrypoint.sh

@@ -1,24 +1,6 @@
 #!/bin/bash
 # =============================================================================
 # entrypoint.sh  –  Label Studio wrapper for Hugging Face Spaces
-#
-# What this script does:
-#   1. On startup  : restores SQLite DB + exports from S3
-#   2. After restore: verifies DB integrity before starting Label Studio
-#   3. Starts Label Studio in the background
-#   4. Every 60 s  : checkpoints WAL then syncs /data → S3
-#   5. On SIGTERM  : does one final sync before the pod dies
-#
-# Required HF Space secrets:
-#   BACKUP_S3_BUCKET        – bucket name (no s3:// prefix)
-#   LABEL_STUDIO_USERNAME   – admin username / email
-#   LABEL_STUDIO_PASSWORD   – admin password
-#
-# Optional HF Space secrets (with defaults shown):
-#   BACKUP_S3_PREFIX        – labelstudio
-#   AWS_DEFAULT_REGION      – us-east-1
-#   AWS_ENDPOINT_URL        – https://s3.amazonaws.com   (change for R2/MinIO)
-#   SYNC_INTERVAL_SECONDS   – 60
 # =============================================================================
 
 set -euo pipefail
@@ -31,27 +13,31 @@ ENDPOINT="${AWS_ENDPOINT_URL:-https://s3.amazonaws.com}"
 SYNC_INTERVAL="${SYNC_INTERVAL_SECONDS:-60}"
 
 DATA_DIR=/data
-# Label Studio uses label_studio.sqlite3 (not .db) — confirmed from live logs
 DB_FILE="$DATA_DIR/label_studio.sqlite3"
+SAFE_BACKUP_FILE="$DATA_DIR/ls_safe_backup.sqlite3"
 S3_PATH="s3://${BUCKET}/${PREFIX}"
 
 export AWS_DEFAULT_REGION="$REGION"
 
-# ------------- helper: WAL checkpoint + S3 upload --------------------------
+# ------------- helper: Atomic SQLite Backup + S3 upload --------------------
 sync_to_s3() {
     local reason="${1:-periodic}"
     echo "[sync] Starting $reason sync to $S3_PATH/ ..."
 
-    # Checkpoint the WAL so S3 always receives a single self-consistent file.
-    # TRUNCATE: folds all WAL frames into the main DB file and zeros the WAL.
-    # This briefly blocks writers (not readers) — safe for ≤10 concurrent users.
+    # 1. Create a safe, atomic backup of the SQLite DB. 
+    # This prevents the race condition of syncing live/journal files.
     if [ -f "$DB_FILE" ]; then
-        sqlite3 "$DB_FILE" "PRAGMA wal_checkpoint(TRUNCATE);" 2>/dev/null || \
-            echo "[sync] WAL checkpoint skipped (DB may be locked) – proceeding anyway"
+        sqlite3 "$DB_FILE" ".backup '$SAFE_BACKUP_FILE'" 2>/dev/null || \
+            echo "[sync] Warning: SQLite backup failed, syncing rest of data."
     fi
 
+    # 2. Sync to S3, explicitly excluding the live database and its lock/journal files
     aws s3 sync "$DATA_DIR/" "$S3_PATH/" \
         --endpoint-url "$ENDPOINT" \
+        --exclude "label_studio.sqlite3" \
+        --exclude "label_studio.sqlite3-journal" \
+        --exclude "label_studio.sqlite3-wal" \
+        --exclude "label_studio.sqlite3-shm" \
         --no-progress \
         && echo "[sync] $reason sync complete" \
         || echo "[sync] WARNING: $reason sync failed – will retry next cycle"
@@ -79,22 +65,28 @@ echo "============================================================"
 mkdir -p "$DATA_DIR"
 
 echo "[restore] Syncing data DOWN from $S3_PATH/ ..."
-# Use || true so a fresh bucket (nothing to restore) doesn't abort the script
 aws s3 sync "$S3_PATH/" "$DATA_DIR/" \
     --endpoint-url "$ENDPOINT" \
     --no-progress \
     && echo "[restore] Restore complete" \
     || echo "[restore] Restore returned non-zero (may be first run) – continuing"
 
-# ------------- 2. DB integrity check before starting LS -------------------
+# ------------- 2. Restore DB & Integrity Check ----------------------------
+# If we have a safe backup from a previous session, make it the live DB
+if [ -f "$SAFE_BACKUP_FILE" ]; then
+    echo "[restore] Overwriting live DB with safe backup from S3..."
+    cp "$SAFE_BACKUP_FILE" "$DB_FILE"
+fi
+
 if [ -f "$DB_FILE" ]; then
     echo "[integrity] Checking DB integrity ..."
-    INTEGRITY=$(sqlite3 "$DB_FILE" "PRAGMA integrity_check;" 2>&1 | head -1)
+    # The `|| true` prevents set -e from silently crashing the script if it fails
+    INTEGRITY=$(sqlite3 "$DB_FILE" "PRAGMA integrity_check;" 2>&1 | head -1 || true)
+    
     if [ "$INTEGRITY" != "ok" ]; then
         echo "[integrity] ERROR: DB integrity check failed: $INTEGRITY"
         echo "[integrity] The restored database is corrupt."
         echo "[integrity] → Check your S3 bucket for a healthy backup."
-        echo "[integrity] → If you have bucket versioning enabled, restore an earlier version."
         exit 1
     fi
     echo "[integrity] DB is healthy: $INTEGRITY"
@@ -111,21 +103,17 @@ label-studio user create \
     --preserve-case 2>/dev/null || true
 
 # ------------- 4. start Label Studio in the background --------------------
-# Use $SPACE_HOST exactly as the official HF Dockerfile does — this is what
-# makes HF's reverse proxy route traffic correctly to the container.
 echo "[start] Starting Label Studio ..."
 label-studio --host="$SPACE_HOST" &
 LS_PID=$!
 echo "[start] Label Studio PID: $LS_PID"
 
-# Register shutdown handler NOW (after we have LS_PID)
 trap shutdown_handler EXIT INT TERM
 
 # ------------- 5. periodic sync loop (runs forever) -----------------------
 echo "[loop] Entering sync loop (every ${SYNC_INTERVAL}s) ..."
 while true; do
     sleep "$SYNC_INTERVAL"
-    # Verify LS is still alive before syncing
     if ! kill -0 "$LS_PID" 2>/dev/null; then
         echo "[loop] Label Studio process has exited unexpectedly – stopping container"
         exit 1