File size: 1,496 Bytes
8f46582 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | #!/usr/bin/env bash
# Kill old coupled-metric diag runs; launch 8 split-gate arms (GPUs 0-7).
set -uo pipefail
cd /egr/research-slim/ghoshavr/reasoning-by-superposition-main
export WANDB_MODE=offline
TORCH=/egr/research-slim/ghoshavr/conda-envs/superposition/bin/torchrun
PY=/egr/research-slim/ghoshavr/conda-envs/superposition/bin/python
mkdir -p logs figs/diag_L15
echo "=== killing old diag_L15 runs ==="
pkill -9 -f 'run.py args/diag_L15_' 2>/dev/null || true
pkill -9 -f 'plot_diag_L15.py' 2>/dev/null || true
sleep 3
"$PY" scripts/make_diag_split_configs.py
ARMS=(
diag_L15_promF095_btCE050
diag_L15_promF095_btCE090
diag_L15_promF095_btCE095
diag_L15_promF099_btCE090
diag_L15_promF095_btF095
diag_L15_promCE090_btCE090
diag_L15_promF095_btNONE
diag_L15_promF099_btCE050
)
for i in "${!ARMS[@]}"; do
name="${ARMS[$i]}"
port=$((29900 + i))
CUDA_VISIBLE_DEVICES="$i" setsid nohup "$TORCH" \
--standalone --nnodes 1 --nproc_per_node 1 --master_port "$port" \
run.py "args/${name}.yaml" > "logs/${name}.log" 2>&1 < /dev/null &
echo "gpu$i port$port $name"
done
setsid nohup "$PY" scripts/plot_diag_L15.py --watch 120 \
> logs/plot_diag_L15.log 2>&1 < /dev/null &
echo "plot watcher $!"
sleep 150
echo '=== status ==='
for name in "${ARMS[@]}"; do
echo "--- $name"
grep -E 'acc-stage|train epoch|Traceback|Error|nohup' "logs/${name}.log" | tail -4 || true
done
nvidia-smi --query-gpu=index,memory.used,utilization.gpu --format=csv,noheader || true
|