File size: 4,914 Bytes
c8d5563
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
#!/usr/bin/env bash
# AdaSpeVLA pipeline: preflight (short train + resume) -> formal Action 40k -> 3 timing runs (from 20k/30k/40k).
# Run on already-chosen GPUs (CUDA_VISIBLE_DEVICES); launched by wait_gpus_and_launch.sh.
# Stops at the first failure; formal stages run only after the preflight passes.
set -euo pipefail

ROOT=$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)  # repo root: this script is <root>/adaspe_train/
TASK=${TASK:-bread}
source "$ROOT/adaspe_train/names.sh"  # TAG, JOB_BASE from VARIANT
RUN=${RUN:-$ROOT/test/franka/$TAG}
PY=$ROOT/external/openpi/.venv/bin/python
TRAIN="$PY $ROOT/adaspe_train/train.py"
mkdir -p "$RUN"
cd "$ROOT/external/openpi"

export TMPDIR=$ROOT/.cache/tmp HF_HUB_OFFLINE=1 HF_LEROBOT_HOME=$ROOT/data/lerobot
export HF_HOME=$ROOT/.cache/huggingface XLA_PYTHON_CLIENT_MEM_FRACTION=${XLA_PYTHON_CLIENT_MEM_FRACTION:-0.9}
export OPENPI_DATA_HOME=$ROOT/.cache/openpi JAX_COMPILATION_CACHE_DIR=$ROOT/.cache/jax
# Keep cluster modules, PYTHONPATH and ~/.local site-packages out of the uv venv.
unset PYTHONPATH LD_LIBRARY_PATH; export PYTHONNOUSERSITE=1 PYTHONUNBUFFERED=1 WANDB_MODE=disabled
mkdir -p "$TMPDIR"

NGPU=$(awk -F, '{print NF}' <<<"${CUDA_VISIBLE_DEVICES:?set by launcher}")
FSDP=(--fsdp_devices "$NGPU")
status() { echo "$(date -Is) $*" | tee -a "$RUN/pipeline_status.log"; }
finite_losses() {  # every logged loss in the log is finite
  ! grep -E "loss=" "$1" | grep -qiE "loss=(nan|inf)" && grep -qE "loss=" "$1"
}

status "start on GPUs $CUDA_VISIBLE_DEVICES (fsdp=$NGPU); git=$(git -C "$ROOT/external/openpi" rev-parse --short HEAD)"
nvidia-smi --query-gpu=index,name,memory.used,memory.total --format=csv > "$RUN/gpus_at_start.csv"
# The 2 h dev chain ($JOB_BASE-dev-$TASK) trains the same formal dirs: stop it first (a no-op inside the dev link).
source "$ROOT/adaspe_train/dev_bridge.sh"
[[ -n "${PREFLIGHT_ONLY:-}" ]] || stop_dev_chain

# A separate dev preflight (slurm/preflight.slurm, job $JOB_BASE-pf-$TASK) may be running: wait so we never share its dir.
if [[ -z "${PREFLIGHT_ONLY:-}" && ! -f "$RUN/preflight_passed" ]]; then
  dev_pf() { squeue -h -u "$USER" -n "$JOB_BASE-pf-$TASK" -t RUNNING -o %A 2>/dev/null || true; }
  if [[ -n "$(dev_pf)" ]]; then
    status "waiting for the running dev preflight $JOB_BASE-pf-$TASK"
    while [[ -n "$(dev_pf)" ]]; do sleep 60; done
  fi
fi

# 1) Preflight: 30 steps, checkpoint at 15/30, then resume to 40.
if [[ ! -f "$RUN/preflight_passed" ]]; then
  status "preflight: train 30 steps"
  $TRAIN ${TAG}_action --exp_name preflight --overwrite \
    --num_train_steps 30 --save_interval 15 --log_interval 5 "${FSDP[@]}" > "$RUN/preflight_train.log" 2>&1
  finite_losses "$RUN/preflight_train.log" || { status "preflight FAILED: non-finite loss"; exit 1; }
  status "preflight: resume to 40"
  $TRAIN ${TAG}_action --exp_name preflight --resume \
    --num_train_steps 40 --save_interval 20 --log_interval 5 "${FSDP[@]}" > "$RUN/preflight_resume.log" 2>&1
  finite_losses "$RUN/preflight_resume.log" || { status "preflight FAILED after resume"; exit 1; }
  # A resumed run starts at step 30: it must log step 35 and never step 0.
  { grep -q "Step 35:" "$RUN/preflight_resume.log" && ! grep -q "Step 0:" "$RUN/preflight_resume.log"; } \
    || { status "preflight FAILED: resume did not continue from step 30"; exit 1; }
  touch "$RUN/preflight_passed"
  rm -rf "$ROOT/checkpoints/adaspe/${TAG}_action/preflight"  # ~50 GB/ckpt; logs kept
  status "preflight passed"
  [[ -n "${PREFLIGHT_ONLY:-}" ]] && { status "stop after preflight (PREFLIGHT_ONLY)"; exit 0; }
fi

# 2) Formal Action stage to 40k (resumes if a checkpoint exists); 10k/20k/30k/40k are kept.
ACT_DIR=$ROOT/checkpoints/adaspe/${TAG}_action/formal
if [[ ! -d "$ACT_DIR/39999/params" ]]; then
  RESUME=(); [[ -d "$ACT_DIR" ]] && RESUME=(--resume)
  status "action: train to 40000 ${RESUME[*]:-}"
  $TRAIN ${TAG}_action --exp_name formal "${RESUME[@]}" "${FSDP[@]}" >> "$RUN/action.log" 2>&1
  for s in 20000 30000 39999; do  # OpenPI saves the last step as num_train_steps-1
    [[ -d "$ACT_DIR/$s/params" ]] || { status "action FAILED: no step-$s params"; exit 1; }
  done
fi
status "action done"
[[ -n "${ACTION_ONLY:-}" ]] && { status "stop after action (ACTION_ONLY; timing via run_timing_one.sh)"; exit 0; }

# 3) Timing stage (timing head only), once per action checkpoint: 20k first (matched budget), then 30k, 40k.
for k in 20 30 40; do
  NAME=${TAG}_timing_from${k}k
  TIM_DIR=$ROOT/checkpoints/adaspe/$NAME/formal
  if [[ ! -d "$TIM_DIR/9999/params" ]]; then
    RESUME=(); [[ -d "$TIM_DIR" ]] && RESUME=(--resume)
    status "timing from ${k}k: train to 10000 ${RESUME[*]:-}"
    $TRAIN $NAME --exp_name formal "${RESUME[@]}" "${FSDP[@]}" >> "$RUN/timing_from${k}k.log" 2>&1
    [[ -d "$TIM_DIR/9999/params" ]] || { status "timing from ${k}k FAILED: no step-9999 params"; exit 1; }
  fi
done
status "completed"