File size: 2,144 Bytes
8f46582 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | #!/usr/bin/env bash
# Final L15 + L20 sweep: 7 arms, one per GPU on devices 1-7.
#
# gpu1 L15 frontier-BT full BPTT 50k <- W=1 ablation reference (L15)
# gpu2 L15 frontier-BT W=1 50k <- W=1 ablation test (L15)
# gpu3 L15 ce_score-BT full BPTT 50k <- loss-gated BT (L15)
# gpu4 L20 frontier-BT full BPTT 50k <- W=1 ablation reference (L20)
# gpu5 L20 frontier-BT W=1 50k <- W=1 ablation test (L20)
# gpu6 L20 ce_score-BT full BPTT 50k <- loss-gated BT (L20)
# gpu7 L20 frontier-BT W=1 100k <- data ablation vs gpu5
#
# setsid so the jobs survive teardown of the launching shell (earlier L15 runs were
# SIGKILLed when the session was reaped).
set -uo pipefail
cd /egr/research-slim/ghoshavr/reasoning-by-superposition-main
export WANDB_MODE=offline
TORCH=/egr/research-slim/ghoshavr/conda-envs/superposition/bin/torchrun
mkdir -p logs
launch () { # $1=gpu $2=port $3=config-basename
CUDA_VISIBLE_DEVICES="$1" setsid nohup "$TORCH" \
--standalone --nnodes 1 --nproc_per_node 1 --master_port "$2" \
run.py "args/$3.yaml" > "logs/$3.log" 2>&1 < /dev/null &
echo "gpu$1 port$2 $3"
}
launch 1 29701 final_L15_frontier_bt_full
launch 2 29702 final_L15_frontier_bt_w1
launch 3 29703 final_L15_ce_bt_full
launch 4 29704 final_L20_frontier_bt_full
launch 5 29705 final_L20_frontier_bt_w1
launch 6 29706 final_L20_ce_bt_full
launch 7 29707 final_L20_frontier_bt_w1_100k
echo
echo "launched 7; waiting 240s for dataset load + first epoch..."
sleep 240
echo '=== procs ==='
pgrep -af 'run.py args/final_' | grep -c torchrun || true
echo '=== GPUs ==='
nvidia-smi --query-gpu=index,memory.used,utilization.gpu --format=csv,noheader -i 1,2,3,4,5,6,7 || true
for f in logs/final_L15_frontier_bt_full logs/final_L15_frontier_bt_w1 \
logs/final_L15_ce_bt_full logs/final_L20_frontier_bt_full \
logs/final_L20_frontier_bt_w1 logs/final_L20_ce_bt_full \
logs/final_L20_frontier_bt_w1_100k; do
echo "--- $f"
grep -E 'acc-stage|Traceback|Error|error|train epoch|stage' "$f.log" 2>/dev/null | tail -4 || true
done
|