File size: 1,211 Bytes
20c62e0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | #!/bin/bash
# Grid 10 - is depth failing for OPTIMISATION reasons? This must be settled
# before the depth-scaling test means anything.
#
# At 8 digits a 6-layer/384 model reached 99.9%, while 10-layer/512 reached
# 66.7% with more training. Deeper transformers usually need a lower learning
# rate and longer warmup; if that is the cause, the depth sweep at a single LR
# would fake a "depth does not help" result.
#SBATCH -J g10
#SBATCH -p gpu-h100
#SBATCH --gpus=h100:1
#SBATCH -c 8
#SBATCH --mem=64g
#SBATCH -t 10:00:00
#SBATCH -o logs/g10-%A_%a.out
#SBATCH --array=0-35%12
source /etc/profile
SCR=/scratch/zt1/project/msml612/user/$USER
module load python/3.10.10
export PYTHONPATH=
PY=$SCR/hf_env/bin/python
cd $SCR/pcmd/src
I=$SLURM_ARRAY_TASK_ID
LAYERS=(6 10); LR=(3e-4 1e-4 5e-5); WARM=(500 4000); SEEDS=(0 1 2)
L=${LAYERS[$((I / 18))]}
R=${LR[$(((I % 18) / 6))]}
W=${WARM[$(((I % 6) / 3))]}
S=${SEEDS[$((I % 3))]}
NAME="g10-L$L-lr$R-w$W-s$S"
echo "=== $NAME ==="
$PY train_add.py --mode full --digits 10 --seq_len 34 --seed $S \
--n_train 500000 --n_eval 400 --eval_bs 100 \
--d 384 --layers $L --heads 6 --bs 128 --steps 40000 --lr $R --warmup $W \
--K 4 --lam 1.0 --out $SCR/pcmd/runs/$NAME
|