scripts/ops/backup.sh

#!/bin/bash
# /root/scripts/backup.sh — Daily backup + monthly restore test
# Per v4.0 §T45. Backs up Postgres + Neo4j + Qdrant + MinIO → /backups/
# Monthly (1st of month) restores to a test container and verifies.

set -euo pipefail

DATE=$(date +%Y%m%d_%H%M%S)
BACKUP_DIR=/backups/$DATE
KEEP_DAYS=7
LOG=/var/log/backup.log

mkdir -p "$BACKUP_DIR"

log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG"; }

log "=== Backup started ==="

# 1. Postgres
log "Backing up Postgres..."
docker exec rmi-postgres pg_dump -U rmi rmi 2>/dev/null | gzip > "$BACKUP_DIR/postgres.sql.gz"
PG_SIZE=$(du -h "$BACKUP_DIR/postgres.sql.gz" | cut -f1)
log "  ✓ postgres.sql.gz ($PG_SIZE)"

# 2. Neo4j (export to SQL via cypher-shell)
log "Backing up Neo4j..."
docker exec rmi-neo4j neo4j-admin database dump neo4j --to-path=/tmp 2>/dev/null || true
if docker cp rmi-neo4j:/tmp/neo4j.dump "$BACKUP_DIR/" 2>/dev/null; then
    NEO_SIZE=$(du -h "$BACKUP_DIR/neo4j.dump" | cut -f1)
    log "  ✓ neo4j.dump ($NEO_SIZE)"
else
    log "  ⚠ Neo4j dump failed (skipping)"
fi

# 3. Qdrant (snapshot API)
log "Backing up Qdrant..."
for collection in $(curl -s http://localhost:6333/collections | python3 -c "
import json, sys
try:
    d = json.load(sys.stdin)
    for c in d.get('result', {}).get('collections', []):
        print(c['name'])
except: pass
"); do
    SNAP=$(curl -s -X POST "http://localhost:6333/collections/$collection/snapshots" 2>/dev/null | python3 -c "
import json, sys
try: print(json.load(sys.stdin).get('result', {}).get('name', ''))
except: pass
")
    if [ -n "$SNAP" ]; then
        mkdir -p "$BACKUP_DIR/qdrant/$collection"
        docker cp "rmi-qdrant:/qdrant/snapshots/$collection/$SNAP" "$BACKUP_DIR/qdrant/$collection/$SNAP" 2>/dev/null || true
        log "  ✓ qdrant/$collection/$SNAP"
    fi
done

# 4. MinIO (rclone → local backup dir)
log "Backing up MinIO..."
if command -v rclone >/dev/null 2>&1; then
    rclone sync minio:rmi-assets "$BACKUP_DIR/minio/" --transfers 4 2>>"$LOG" || log "  ⚠ rclone sync partial"
else
    # Fallback: mc mirror
    docker run --rm -v /data:/data --network rmi_network minio/mc:latest \
        mirror --overwrite minio/rmi-assets "$BACKUP_DIR/minio/" 2>>"$LOG" || log "  ⚠ mc mirror failed"
fi

# 5. GlitchTip (Postgres dump)
log "Backing up GlitchTip..."

# 5b. System configs (/etc + /root) - surgical backup for DR
log "Backing up system configs..."
tar -czf "$BACKUP_DIR/etc_root.tar.gz" \
    --exclude='/root/.cache' \
    --exclude='/root/.ollama' \
    --exclude='/root/.hermes/sessions' \
    --exclude='/root/.cargo' \
    --exclude='/root/.rustup' \
    --exclude='/root/.local/share/cargo' \
    /etc/prometheus \
    /root/scripts /root/.hermes /root/.ssh /root/.bashrc /root/.profile \
    /root/backend/.env \
    2>/dev/null || true
if [ -f "$BACKUP_DIR/etc_root.tar.gz" ]; then
    ER_SIZE=$(du -h "$BACKUP_DIR/etc_root.tar.gz" | cut -f1)
    log "  ✓ etc_root.tar.gz ($ER_SIZE)"
    if tar -tzf "$BACKUP_DIR/etc_root.tar.gz" 2>/dev/null | grep -q "etc/prometheus/prometheus.yml"; then
        log "  ✓ etc_root.tar.gz valid"
    fi
else
    log "  ⚠ etc_root.tar.gz creation failed"
fi
log "Backing up GlitchTip..."
docker exec rmi-glitchtip-db pg_dump -U glitchtip glitchtip 2>/dev/null | gzip > "$BACKUP_DIR/glitchtip.sql.gz" || true

# 6. Rotation
log "Rotating old backups (keep ${KEEP_DAYS} days)..."
find /backups -maxdepth 1 -type d -mtime +$KEEP_DAYS -exec rm -rf {} \; 2>/dev/null || true
REMAINING=$(ls -1d /backups/*/ 2>/dev/null | wc -l)
TOTAL_SIZE=$(du -sh /backups 2>/dev/null | cut -f1)

log "=== Backup complete: $BACKUP_DIR ==="
log "  Remaining snapshots: $REMAINING"
log "  Total backup size: $TOTAL_SIZE"

# 7. Monthly restore test (1st of month)
if [ "$(date +%d)" = "01" ]; then
    log "=== Monthly restore test ==="
    /root/scripts/restore_test.sh 2>&1 | tee -a "$LOG"
fi

# Push to ntfy if running
if curl -s -o /dev/null -w '%{http_code}' http://localhost:9080 2>/dev/null | grep -q '^2'; then
    curl -s -d "Backup complete: $BACKUP_DIR ($TOTAL_SIZE, $REMAINING snapshots)" \
        http://localhost:9080/rmi-backups 2>/dev/null || true
fi