backup_checkpoints.sh

#!/bin/bash
# Backup script: copies training checkpoints to a dated archive dir.
# Run: nohup ./backup_checkpoints.sh &
SRC=/opt/sn32-data/bootstrap
DST=/opt/sn32-data/checkpoint_backup/$(date +%Y%m%d_%H%M%S)
mkdir -p "$DST"
echo "Backing up checkpoints to $DST"

# Check for existing checkpoints
for f in "$SRC"/ood_qwen3_ckpt_step*.pt "$SRC"/ood_qwen3_results.json; do
  [ -f "$f" ] && cp "$f" "$DST/"
done

# Watch for new/modified checkpoint files
while inotifywait -q -e close_write,moved_to "$SRC" 2>/dev/null; do
  for f in "$SRC"/ood_qwen3_ckpt_step*.pt; do
    [ -f "$f" ] && cp "$f" "$DST/"
  done
done