| # Start/stop/inspect the Hugging Face checkpoint auto-sync (scripts/hf_ckpt_sync.py) on the GPU pod. | |
| # bash scripts/hf_ckpt_sync.sh start # start the 3-hourly loop if not already running (safe to re-run, e.g. after a pod restart) | |
| # bash scripts/hf_ckpt_sync.sh status # pid, last log lines, last uploaded file | |
| # bash scripts/hf_ckpt_sync.sh once # one immediate sync cycle (fails if the loop is mid-upload, lock-protected) | |
| # bash scripts/hf_ckpt_sync.sh stop | |
| # No cron on the pod: after a pod restart run "start" again (the training driver must be restarted separately). | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| PY=${HFSYNC_PY:-$HOME/hfsync-venv/bin/python}; PIDF=logs/hf_sync.pid; LOG=logs/hf_sync.log | |
| mkdir -p logs | |
| running() { [ -f "$PIDF" ] && kill -0 "$(cat "$PIDF")" 2>/dev/null; } | |
| case "${1:-status}" in | |
| start) if running; then echo "already running pid $(cat $PIDF)"; exit 0; fi | |
| [ -s "$HOME/.cache/huggingface/token" ] || { echo "no HF token at ~/.cache/huggingface/token"; exit 1; } | |
| [ -x "$PY" ] || { /opt/conda/bin/python -m venv "$HOME/hfsync-venv" && "$HOME/hfsync-venv/bin/pip" install -q -U huggingface_hub; } | |
| SETSID=""; command -v setsid >/dev/null && SETSID=setsid | |
| IONICE=""; command -v ionice >/dev/null && IONICE="ionice -c3" | |
| $SETSID nohup nice -n 19 $IONICE "$PY" -u scripts/hf_ckpt_sync.py --loop >> "$LOG" 2>&1 < /dev/null & | |
| sleep 3; running && echo "started pid $(cat $PIDF), log $LOG" || { echo "failed to start; see $LOG"; tail -5 "$LOG"; exit 1; } ;; | |
| stop) running && kill "$(cat $PIDF)" && echo stopped || echo "not running" ;; | |
| once) nice -n 19 ionice -c3 "$PY" -u scripts/hf_ckpt_sync.py >> "$LOG" 2>&1; tail -3 "$LOG" ;; | |
| status) running && echo "running pid $(cat $PIDF)" || echo "NOT running (start: bash scripts/hf_ckpt_sync.sh start)" | |
| tail -n 5 "$LOG" 2>/dev/null; cat logs/hf_sync_state.json 2>/dev/null; echo ;; | |
| *) sed -n '2,7p' "$0"; exit 1 ;; | |
| esac | |