File size: 1,359 Bytes
20c62e0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 | #!/bin/bash
# Grid 7 - LEARNABILITY GATE for long addition, before any parallelism claim.
#
# At 14-16 digits every method scored 0 even at FULLY SEQUENTIAL decoding, so
# the models had not learned addition at all. Any statement about parallel
# decoding there would be vacuous. This sweeps capacity and training length to
# find where the task becomes learnable; only then does measuring parallelism
# mean anything.
#SBATCH -J g7
#SBATCH -p gpu-h100
#SBATCH --gpus=h100:1
#SBATCH -c 8
#SBATCH --mem=64g
#SBATCH -t 10:00:00
#SBATCH -o logs/g7-%A_%a.out
#SBATCH --array=0-23%12
source /etc/profile
SCR=/scratch/zt1/project/msml612/user/$USER
module load python/3.10.10
export PYTHONPATH=
PY=$SCR/hf_env/bin/python
cd $SCR/pcmd/src
I=$SLURM_ARRAY_TASK_ID
# capacity x training length, on the two settings that failed outright
CFG=(
"12 8 512 60000" "12 12 512 60000" "12 8 512 120000"
"16 8 512 60000" "16 12 512 60000" "16 12 768 120000"
)
MODES=(mdlm full)
C=${CFG[$((I / 4))]}
M=${MODES[$(((I % 4) / 2))]}
S=$((I % 2))
read -r D LY DM ST <<< "$C"
SL=$(( 3 * D + 4 ))
NAME="g7-d$D-L$LY-dm$DM-st$ST-$M-s$S"
echo "=== $NAME ==="
$PY train_add.py --mode $M --digits $D --seq_len $SL --seed $S \
--n_train 500000 --n_eval 400 --eval_bs 100 \
--d $DM --layers $LY --heads 8 --bs 128 --steps $ST --lr 3e-4 \
--K 4 --lam 1.0 --out $SCR/pcmd/runs/$NAME
|