Buckets:
| # ============================================================== | |
| # GCP Checkpoint Sync Script | |
| # ============================================================== | |
| # Periodically uploads checkpoints to GCS for safety. | |
| # Run this in a separate tmux pane or terminal while training. | |
| # | |
| # If the spot VM gets preempted, your checkpoints are safe in GCS. | |
| # When you create a new VM, just pull them down and resume. | |
| # | |
| # Usage: | |
| # bash scripts/gcp/sync_checkpoints.sh # Default: every 30 min | |
| # bash scripts/gcp/sync_checkpoints.sh --interval 600 # Every 10 min | |
| # ============================================================== | |
| set -e | |
| INTERVAL=1800 # 30 minutes default | |
| BUCKET="${GCS_BUCKET:-}" | |
| while [[ $# -gt 0 ]]; do | |
| case $1 in | |
| --interval) INTERVAL="$2"; shift 2 ;; | |
| --bucket) BUCKET="$2"; shift 2 ;; | |
| *) echo "Unknown arg: $1"; exit 1 ;; | |
| esac | |
| done | |
| if [ -z "$BUCKET" ]; then | |
| echo "ERROR: Set GCS_BUCKET environment variable or pass --bucket" | |
| echo " export GCS_BUCKET=your-bucket-name" | |
| exit 1 | |
| fi | |
| GCS_PATH="gs://${BUCKET}/Train_Advanced_Lipsync_Model/checkpoints/" | |
| LOCAL_PATH=~/Train_Advanced_Lipsync_Model/checkpoints/ | |
| echo "============================================" | |
| echo " Checkpoint Sync to GCS" | |
| echo "============================================" | |
| echo " Local: $LOCAL_PATH" | |
| echo " Remote: $GCS_PATH" | |
| echo " Interval: ${INTERVAL}s" | |
| echo "============================================" | |
| SYNC_COUNT=0 | |
| while true; do | |
| if [ -d "$LOCAL_PATH" ] && [ "$(ls -A $LOCAL_PATH 2>/dev/null)" ]; then | |
| SYNC_COUNT=$((SYNC_COUNT + 1)) | |
| TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S') | |
| echo "[$TIMESTAMP] Sync #$SYNC_COUNT — uploading checkpoints..." | |
| gsutil -m rsync -r "$LOCAL_PATH" "$GCS_PATH" | |
| # Also sync TensorBoard logs | |
| if [ -d ~/Train_Advanced_Lipsync_Model/logs/ ]; then | |
| gsutil -m rsync -r ~/Train_Advanced_Lipsync_Model/logs/ \ | |
| "gs://${BUCKET}/Train_Advanced_Lipsync_Model/logs/" | |
| fi | |
| echo "[$TIMESTAMP] Sync complete." | |
| else | |
| echo "[$(date '+%H:%M:%S')] No checkpoints yet, waiting..." | |
| fi | |
| sleep $INTERVAL | |
| done | |
Xet Storage Details
- Size:
- 2.22 kB
- Xet hash:
- 69c5cbbb0ae42c5af5f43a982f69f1364227168d266fdf7630217456588fe274
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.