scripts/start/persistence-start.sh

#!/bin/bash

set -e

# Log function
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] [PERSISTENCE] $*"
}

log "Persistence service startup initiated"

# Check if persistence is properly configured
if [[ -z "$HF_TOKEN" || -z "$DATASET_ID" ]]; then
    log "ERROR: Persistence enabled but missing required configuration"
    log "Required environment variables: HF_TOKEN, DATASET_ID"
    log "HF_TOKEN: ${HF_TOKEN:+configured}"
    log "DATASET_ID: ${DATASET_ID:-not set}"
    exit 1
fi

log "Persistence configuration detected"
log "HF_TOKEN: ${HF_TOKEN:0:10}..."
log "DATASET_ID: $DATASET_ID"

# Set up Hugging Face cache directory environment variables
export HF_HOME="/home/user/.cache/huggingface"
export HUGGINGFACE_HUB_CACHE="/home/user/.cache/huggingface"
mkdir -p "$HF_HOME"
log "Hugging Face cache directory configured: $HF_HOME"

# PHASE 1: Synchronous data restoration (BLOCKING)
log "=== PHASE 1: SYNCHRONOUS DATA RESTORATION ==="
log "Performing synchronous data restoration - this MUST complete before any services start"

if "$(dirname "$(dirname "$0")")/utils/persistence.sh" restore-sync latest; then
    log "✓ Synchronous data restoration completed successfully"
    log "All data has been restored and verified"
else
    log "✗ Synchronous data restoration failed"
    log "ERROR: Cannot proceed with service startup due to data restoration failure"
    log "This prevents data inconsistency issues"
    exit 1
fi

log "=== PHASE 1 COMPLETED: DATA RESTORATION SUCCESSFUL ==="

# PHASE 2: Start persistence daemon (AFTER restoration)
log "=== PHASE 2: STARTING PERSISTENCE DAEMON ==="
log "Starting persistence daemon (data restoration completed)..."
"$(dirname "$(dirname "$0")")/utils/persistence.sh" daemon &
DAEMON_PID=$!
log "Persistence daemon started with PID: $DAEMON_PID"

log "=== PERSISTENCE SERVICE STARTUP COMPLETED ==="
log "✓ Data restoration: COMPLETED"
log "✓ Persistence daemon: RUNNING (PID: $DAEMON_PID)"
log "All dependent services can now start safely"

# Return to main startup script to continue with other services
# The daemon will continue running in the background
log "Persistence service initialization completed, returning to main startup process"