cognaclee's picture
Add files using upload-large-folder tool
c8d5563 verified
Raw History Blame Contribute Delete
4.91 kB
#!/usr/bin/env bash
# AdaSpeVLA pipeline: preflight (short train + resume) -> formal Action 40k -> 3 timing runs (from 20k/30k/40k).
# Run on already-chosen GPUs (CUDA_VISIBLE_DEVICES); launched by wait_gpus_and_launch.sh.
# Stops at the first failure; formal stages run only after the preflight passes.
set -euo pipefail
ROOT=$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) # repo root: this script is <root>/adaspe_train/
TASK=${TASK:-bread}
source "$ROOT/adaspe_train/names.sh" # TAG, JOB_BASE from VARIANT
RUN=${RUN:-$ROOT/test/franka/$TAG}
PY=$ROOT/external/openpi/.venv/bin/python
TRAIN="$PY $ROOT/adaspe_train/train.py"
mkdir -p "$RUN"
cd "$ROOT/external/openpi"
export TMPDIR=$ROOT/.cache/tmp HF_HUB_OFFLINE=1 HF_LEROBOT_HOME=$ROOT/data/lerobot
export HF_HOME=$ROOT/.cache/huggingface XLA_PYTHON_CLIENT_MEM_FRACTION=${XLA_PYTHON_CLIENT_MEM_FRACTION:-0.9}
export OPENPI_DATA_HOME=$ROOT/.cache/openpi JAX_COMPILATION_CACHE_DIR=$ROOT/.cache/jax
# Keep cluster modules, PYTHONPATH and ~/.local site-packages out of the uv venv.
unset PYTHONPATH LD_LIBRARY_PATH; export PYTHONNOUSERSITE=1 PYTHONUNBUFFERED=1 WANDB_MODE=disabled
mkdir -p "$TMPDIR"
NGPU=$(awk -F, '{print NF}' <<<"${CUDA_VISIBLE_DEVICES:?set by launcher}")
FSDP=(--fsdp_devices "$NGPU")
status() { echo "$(date -Is) $*" | tee -a "$RUN/pipeline_status.log"; }
finite_losses() { # every logged loss in the log is finite
! grep -E "loss=" "$1" | grep -qiE "loss=(nan|inf)" && grep -qE "loss=" "$1"
}
status "start on GPUs $CUDA_VISIBLE_DEVICES (fsdp=$NGPU); git=$(git -C "$ROOT/external/openpi" rev-parse --short HEAD)"
nvidia-smi --query-gpu=index,name,memory.used,memory.total --format=csv > "$RUN/gpus_at_start.csv"
# The 2 h dev chain ($JOB_BASE-dev-$TASK) trains the same formal dirs: stop it first (a no-op inside the dev link).
source "$ROOT/adaspe_train/dev_bridge.sh"
[[ -n "${PREFLIGHT_ONLY:-}" ]] || stop_dev_chain
# A separate dev preflight (slurm/preflight.slurm, job $JOB_BASE-pf-$TASK) may be running: wait so we never share its dir.
if [[ -z "${PREFLIGHT_ONLY:-}" && ! -f "$RUN/preflight_passed" ]]; then
dev_pf() { squeue -h -u "$USER" -n "$JOB_BASE-pf-$TASK" -t RUNNING -o %A 2>/dev/null || true; }
if [[ -n "$(dev_pf)" ]]; then
status "waiting for the running dev preflight $JOB_BASE-pf-$TASK"
while [[ -n "$(dev_pf)" ]]; do sleep 60; done
fi
fi
# 1) Preflight: 30 steps, checkpoint at 15/30, then resume to 40.
if [[ ! -f "$RUN/preflight_passed" ]]; then
status "preflight: train 30 steps"
$TRAIN ${TAG}_action --exp_name preflight --overwrite \
--num_train_steps 30 --save_interval 15 --log_interval 5 "${FSDP[@]}" > "$RUN/preflight_train.log" 2>&1
finite_losses "$RUN/preflight_train.log" || { status "preflight FAILED: non-finite loss"; exit 1; }
status "preflight: resume to 40"
$TRAIN ${TAG}_action --exp_name preflight --resume \
--num_train_steps 40 --save_interval 20 --log_interval 5 "${FSDP[@]}" > "$RUN/preflight_resume.log" 2>&1
finite_losses "$RUN/preflight_resume.log" || { status "preflight FAILED after resume"; exit 1; }
# A resumed run starts at step 30: it must log step 35 and never step 0.
{ grep -q "Step 35:" "$RUN/preflight_resume.log" && ! grep -q "Step 0:" "$RUN/preflight_resume.log"; } \
|| { status "preflight FAILED: resume did not continue from step 30"; exit 1; }
touch "$RUN/preflight_passed"
rm -rf "$ROOT/checkpoints/adaspe/${TAG}_action/preflight" # ~50 GB/ckpt; logs kept
status "preflight passed"
[[ -n "${PREFLIGHT_ONLY:-}" ]] && { status "stop after preflight (PREFLIGHT_ONLY)"; exit 0; }
fi
# 2) Formal Action stage to 40k (resumes if a checkpoint exists); 10k/20k/30k/40k are kept.
ACT_DIR=$ROOT/checkpoints/adaspe/${TAG}_action/formal
if [[ ! -d "$ACT_DIR/39999/params" ]]; then
RESUME=(); [[ -d "$ACT_DIR" ]] && RESUME=(--resume)
status "action: train to 40000 ${RESUME[*]:-}"
$TRAIN ${TAG}_action --exp_name formal "${RESUME[@]}" "${FSDP[@]}" >> "$RUN/action.log" 2>&1
for s in 20000 30000 39999; do # OpenPI saves the last step as num_train_steps-1
[[ -d "$ACT_DIR/$s/params" ]] || { status "action FAILED: no step-$s params"; exit 1; }
done
fi
status "action done"
[[ -n "${ACTION_ONLY:-}" ]] && { status "stop after action (ACTION_ONLY; timing via run_timing_one.sh)"; exit 0; }
# 3) Timing stage (timing head only), once per action checkpoint: 20k first (matched budget), then 30k, 40k.
for k in 20 30 40; do
NAME=${TAG}_timing_from${k}k
TIM_DIR=$ROOT/checkpoints/adaspe/$NAME/formal
if [[ ! -d "$TIM_DIR/9999/params" ]]; then
RESUME=(); [[ -d "$TIM_DIR" ]] && RESUME=(--resume)
status "timing from ${k}k: train to 10000 ${RESUME[*]:-}"
$TRAIN $NAME --exp_name formal "${RESUME[@]}" "${FSDP[@]}" >> "$RUN/timing_from${k}k.log" 2>&1
[[ -d "$TIM_DIR/9999/params" ]] || { status "timing from ${k}k FAILED: no step-9999 params"; exit 1; }
fi
done
status "completed"