#!/usr/bin/env bash
# Start/stop/inspect the Hugging Face checkpoint auto-sync (scripts/hf_ckpt_sync.py) on the GPU pod.
#   bash scripts/hf_ckpt_sync.sh start    # start the 3-hourly loop if not already running (safe to re-run, e.g. after a pod restart)
#   bash scripts/hf_ckpt_sync.sh status   # pid, last log lines, last uploaded file
#   bash scripts/hf_ckpt_sync.sh once     # one immediate sync cycle (fails if the loop is mid-upload, lock-protected)
#   bash scripts/hf_ckpt_sync.sh stop
# No cron on the pod: after a pod restart run "start" again (the training driver must be restarted separately).
set -u
cd "$(dirname "$0")/.."
PY=${HFSYNC_PY:-$HOME/hfsync-venv/bin/python}; PIDF=logs/hf_sync.pid; LOG=logs/hf_sync.log
mkdir -p logs
running() { [ -f "$PIDF" ] && kill -0 "$(cat "$PIDF")" 2>/dev/null; }
case "${1:-status}" in
  start) if running; then echo "already running pid $(cat $PIDF)"; exit 0; fi
         [ -s "$HOME/.cache/huggingface/token" ] || { echo "no HF token at ~/.cache/huggingface/token"; exit 1; }
         [ -x "$PY" ] || { /opt/conda/bin/python -m venv "$HOME/hfsync-venv" && "$HOME/hfsync-venv/bin/pip" install -q -U huggingface_hub; }
         SETSID=""; command -v setsid >/dev/null && SETSID=setsid
         IONICE=""; command -v ionice >/dev/null && IONICE="ionice -c3"
         $SETSID nohup nice -n 19 $IONICE "$PY" -u scripts/hf_ckpt_sync.py --loop >> "$LOG" 2>&1 < /dev/null &
         sleep 3; running && echo "started pid $(cat $PIDF), log $LOG" || { echo "failed to start; see $LOG"; tail -5 "$LOG"; exit 1; } ;;
  stop)  running && kill "$(cat $PIDF)" && echo stopped || echo "not running" ;;
  once)  nice -n 19 ionice -c3 "$PY" -u scripts/hf_ckpt_sync.py >> "$LOG" 2>&1; tail -3 "$LOG" ;;
  status) running && echo "running pid $(cat $PIDF)" || echo "NOT running (start: bash scripts/hf_ckpt_sync.sh start)"
          tail -n 5 "$LOG" 2>/dev/null; cat logs/hf_sync_state.json 2>/dev/null; echo ;;
  *) sed -n '2,7p' "$0"; exit 1 ;;
esac