File size: 1,327 Bytes
8f46582 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | #!/usr/bin/env bash
# L15 "make it work" sweep: mirror the L10 winning recipe, then vary data and capacity.
# GPUs 2,3 are left alone (live L10 ce095 / ce099 runs).
set -uo pipefail
cd /egr/research-slim/ghoshavr/reasoning-by-superposition-main
export PATH=/egr/research-slim/ghoshavr/conda-envs/superposition/bin:$PATH
export WANDB_MODE=offline
TORCH=/egr/research-slim/ghoshavr/conda-envs/superposition/bin/torchrun
mkdir -p logs
launch () { # gpu port config
CUDA_VISIBLE_DEVICES=$1 setsid nohup "$TORCH" --standalone --nnodes 1 --nproc_per_node 1 \
--master_port "$2" run.py "args/$3.yaml" > "logs/$3.log" 2>&1 < /dev/null &
echo "launched $3 on GPU $1"
}
launch 1 29701 L15_push_2L_ce95_50k
launch 4 29702 L15_push_2L_ce95_100k
launch 5 29703 L15_push_4L_ce95_50k
launch 6 29704 L15_push_4L_ce90_50k
launch 7 29705 L15_push_s0_4L
sleep 90
echo
echo '=== procs ==='
pgrep -af '[r]un.py' | grep -o 'args/L15_push[^ ]*' | sort | uniq -c
echo
echo '=== GPUs ==='
nvidia-smi --query-gpu=index,memory.used,utilization.gpu --format=csv,noheader
echo
for n in L15_push_2L_ce95_50k L15_push_2L_ce95_100k L15_push_4L_ce95_50k L15_push_4L_ce90_50k L15_push_s0_4L; do
echo "--- $n ---"
grep -E 'Traceback|Error|error|missing_keys|unexpected_keys|_IncompatibleKeys|train epoch' "logs/$n.log" 2>/dev/null | head -3
done
|