minhluanquach's picture
download
raw
2.22 kB
#!/bin/bash
# ==============================================================
# GCP Checkpoint Sync Script
# ==============================================================
# Periodically uploads checkpoints to GCS for safety.
# Run this in a separate tmux pane or terminal while training.
#
# If the spot VM gets preempted, your checkpoints are safe in GCS.
# When you create a new VM, just pull them down and resume.
#
# Usage:
# bash scripts/gcp/sync_checkpoints.sh # Default: every 30 min
# bash scripts/gcp/sync_checkpoints.sh --interval 600 # Every 10 min
# ==============================================================
set -e
INTERVAL=1800 # 30 minutes default
BUCKET="${GCS_BUCKET:-}"
while [[ $# -gt 0 ]]; do
case $1 in
--interval) INTERVAL="$2"; shift 2 ;;
--bucket) BUCKET="$2"; shift 2 ;;
*) echo "Unknown arg: $1"; exit 1 ;;
esac
done
if [ -z "$BUCKET" ]; then
echo "ERROR: Set GCS_BUCKET environment variable or pass --bucket"
echo " export GCS_BUCKET=your-bucket-name"
exit 1
fi
GCS_PATH="gs://${BUCKET}/Train_Advanced_Lipsync_Model/checkpoints/"
LOCAL_PATH=~/Train_Advanced_Lipsync_Model/checkpoints/
echo "============================================"
echo " Checkpoint Sync to GCS"
echo "============================================"
echo " Local: $LOCAL_PATH"
echo " Remote: $GCS_PATH"
echo " Interval: ${INTERVAL}s"
echo "============================================"
SYNC_COUNT=0
while true; do
if [ -d "$LOCAL_PATH" ] && [ "$(ls -A $LOCAL_PATH 2>/dev/null)" ]; then
SYNC_COUNT=$((SYNC_COUNT + 1))
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
echo "[$TIMESTAMP] Sync #$SYNC_COUNT — uploading checkpoints..."
gsutil -m rsync -r "$LOCAL_PATH" "$GCS_PATH"
# Also sync TensorBoard logs
if [ -d ~/Train_Advanced_Lipsync_Model/logs/ ]; then
gsutil -m rsync -r ~/Train_Advanced_Lipsync_Model/logs/ \
"gs://${BUCKET}/Train_Advanced_Lipsync_Model/logs/"
fi
echo "[$TIMESTAMP] Sync complete."
else
echo "[$(date '+%H:%M:%S')] No checkpoints yet, waiting..."
fi
sleep $INTERVAL
done

Xet Storage Details

Size:
2.22 kB
·
Xet hash:
69c5cbbb0ae42c5af5f43a982f69f1364227168d266fdf7630217456588fe274

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.