File size: 2,238 Bytes
42c0d23 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | #!/usr/bin/env bash
# scripts/launch.sh — launch a single training run, detached via nohup.
#
# Usage:
# bash scripts/launch.sh [condition] [n_train] [seed] [gpu]
#
# Defaults:
# condition = grokking
# n_train = 500
# seed = 42
# gpu = 0
#
# Survives SSH disconnects. All artifacts in experiments/runs/<run_id>/.
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
cd "${ROOT}"
source "${ROOT}/scripts/lib/nohup_runner.sh"
source "${ROOT}/scripts/lib/env.sh"
resolve_python
CONDITION="${1:-grokking}"
N_TRAIN="${2:-500}"
SEED="${3:-42}"
GPU="${4:-0}"
WANDB_MODE_ARG="${WANDB_MODE:-online}" # online|offline|disabled
EXTRA_ARGS="${EXTRA_ARGS:-}" # extra trainer flags pass-through
# e.g. EXTRA_ARGS="--spurious_rho 0.8 --weight_decay 5e-3"
RUN_TAG="${RUN_TAG:-}" # extra slug inserted into the run_id between
# <condition> and n<N>. Caller-controlled because
# the launcher pre-creates the run_dir before the
# trainer can self-tag.
# e.g. RUN_TAG=spurious08 RUN_TAG=spurious095_ps10
STAMP="$(date -u +%Y%m%d-%H%M%S)"
TAG_INFIX=""
[[ -n "${RUN_TAG}" ]] && TAG_INFIX="_${RUN_TAG}"
RUN_ID="${STAMP}_${CONDITION}${TAG_INFIX}_n${N_TRAIN}_s${SEED}"
RUN_DIR="experiments/runs/${RUN_ID}"
mkdir -p "${RUN_DIR}"/{logs,results,checkpoints,figures}
echo "Launching CausalGrok run"
echo " condition : ${CONDITION}"
echo " n_train : ${N_TRAIN}"
echo " seed : ${SEED}"
echo " GPU : ${GPU}"
echo " run_id : ${RUN_ID}"
# Build the trainer argv. EXTRA_ARGS is intentionally unquoted so it
# word-splits into individual flags (env vars cannot carry arrays).
read -r -a _extra <<<"${EXTRA_ARGS}"
CUDA_VISIBLE_DEVICES="${GPU}" launch_detached "${RUN_DIR}" \
"${PYTHON}" -u -m experiments.causalgrok_baseline \
--condition "${CONDITION}" \
--n_train "${N_TRAIN}" \
--seed "${SEED}" \
--run_dir "${RUN_DIR}" \
--wandb_project causalgrok \
--wandb_mode "${WANDB_MODE_ARG}" \
"${_extra[@]}"
|