File size: 1,110 Bytes
7ed86c3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | #!/bin/bash
# Submit the final run as a CHAIN of dependent jobs instead of one long monolithic request.
# Each job requests a modest wall-time (schedules far more easily as backfill), checkpoints
# every ckpt_every steps, and the trainer auto-resumes from out_dir/last.pt — so a crash or
# timeout only loses at most one checkpoint interval, not the whole run.
#
# Usage: scripts/pretrain/launch_chain.sh [n_jobs] [hours_per_job] [config]
# scripts/pretrain/launch_chain.sh 6 8 configs/pretrain/stoicheia.json # 6 x 8h = 48h budget
set -euo pipefail
N=${1:-6}
HOURS=${2:-8}
CONFIG=${3:-configs/pretrain/stoicheia.json}
cd "$(dirname "$0")/../.."
PREV=""
JOBIDS=()
for i in $(seq 1 "$N"); do
if [ -z "$PREV" ]; then DEP=""; else DEP="--dependency=afterany:$PREV"; fi
J=$(sbatch --parsable --time="${HOURS}:00:00" --job-name=gcb-final $DEP \
-o "logs/final-%j.out" -e "logs/final-%j.out" \
scripts/slurm/pretrain_final.sbatch "$CONFIG")
echo "chain job $i/$N: $J (dep: ${PREV:-none})"
JOBIDS+=("$J")
PREV=$J
done
echo "chain: ${JOBIDS[*]}"
echo "total wall-time budget: $((N * HOURS))h"
|