#!/usr/bin/env bash
# Final L15 + L20 sweep: 7 arms, one per GPU on devices 1-7.
#
#   gpu1  L15 frontier-BT  full BPTT   50k   <- W=1 ablation reference (L15)
#   gpu2  L15 frontier-BT  W=1         50k   <- W=1 ablation test      (L15)
#   gpu3  L15 ce_score-BT  full BPTT   50k   <- loss-gated BT          (L15)
#   gpu4  L20 frontier-BT  full BPTT   50k   <- W=1 ablation reference (L20)
#   gpu5  L20 frontier-BT  W=1         50k   <- W=1 ablation test      (L20)
#   gpu6  L20 ce_score-BT  full BPTT   50k   <- loss-gated BT          (L20)
#   gpu7  L20 frontier-BT  W=1        100k   <- data ablation vs gpu5
#
# setsid so the jobs survive teardown of the launching shell (earlier L15 runs were
# SIGKILLed when the session was reaped).
set -uo pipefail
cd /egr/research-slim/ghoshavr/reasoning-by-superposition-main
export WANDB_MODE=offline
TORCH=/egr/research-slim/ghoshavr/conda-envs/superposition/bin/torchrun
mkdir -p logs

launch () {  # $1=gpu  $2=port  $3=config-basename
  CUDA_VISIBLE_DEVICES="$1" setsid nohup "$TORCH" \
    --standalone --nnodes 1 --nproc_per_node 1 --master_port "$2" \
    run.py "args/$3.yaml" > "logs/$3.log" 2>&1 < /dev/null &
  echo "gpu$1  port$2  $3"
}

launch 1 29701 final_L15_frontier_bt_full
launch 2 29702 final_L15_frontier_bt_w1
launch 3 29703 final_L15_ce_bt_full
launch 4 29704 final_L20_frontier_bt_full
launch 5 29705 final_L20_frontier_bt_w1
launch 6 29706 final_L20_ce_bt_full
launch 7 29707 final_L20_frontier_bt_w1_100k

echo
echo "launched 7; waiting 240s for dataset load + first epoch..."
sleep 240

echo '=== procs ==='
pgrep -af 'run.py args/final_' | grep -c torchrun || true
echo '=== GPUs ==='
nvidia-smi --query-gpu=index,memory.used,utilization.gpu --format=csv,noheader -i 1,2,3,4,5,6,7 || true
for f in logs/final_L15_frontier_bt_full logs/final_L15_frontier_bt_w1 \
         logs/final_L15_ce_bt_full logs/final_L20_frontier_bt_full \
         logs/final_L20_frontier_bt_w1 logs/final_L20_ce_bt_full \
         logs/final_L20_frontier_bt_w1_100k; do
  echo "--- $f"
  grep -E 'acc-stage|Traceback|Error|error|train epoch|stage' "$f.log" 2>/dev/null | tail -4 || true
done