#!/usr/bin/env bash
# Kill old coupled-metric diag runs; launch 8 split-gate arms (GPUs 0-7).
set -uo pipefail
cd /egr/research-slim/ghoshavr/reasoning-by-superposition-main
export WANDB_MODE=offline
TORCH=/egr/research-slim/ghoshavr/conda-envs/superposition/bin/torchrun
PY=/egr/research-slim/ghoshavr/conda-envs/superposition/bin/python
mkdir -p logs figs/diag_L15

echo "=== killing old diag_L15 runs ==="
pkill -9 -f 'run.py args/diag_L15_' 2>/dev/null || true
pkill -9 -f 'plot_diag_L15.py' 2>/dev/null || true
sleep 3

"$PY" scripts/make_diag_split_configs.py

ARMS=(
  diag_L15_promF095_btCE050
  diag_L15_promF095_btCE090
  diag_L15_promF095_btCE095
  diag_L15_promF099_btCE090
  diag_L15_promF095_btF095
  diag_L15_promCE090_btCE090
  diag_L15_promF095_btNONE
  diag_L15_promF099_btCE050
)

for i in "${!ARMS[@]}"; do
  name="${ARMS[$i]}"
  port=$((29900 + i))
  CUDA_VISIBLE_DEVICES="$i" setsid nohup "$TORCH" \
    --standalone --nnodes 1 --nproc_per_node 1 --master_port "$port" \
    run.py "args/${name}.yaml" > "logs/${name}.log" 2>&1 < /dev/null &
  echo "gpu$i  port$port  $name"
done

setsid nohup "$PY" scripts/plot_diag_L15.py --watch 120 \
  > logs/plot_diag_L15.log 2>&1 < /dev/null &
echo "plot watcher $!"

sleep 150
echo '=== status ==='
for name in "${ARMS[@]}"; do
  echo "--- $name"
  grep -E 'acc-stage|train epoch|Traceback|Error|nohup' "logs/${name}.log" | tail -4 || true
done
nvidia-smi --query-gpu=index,memory.used,utilization.gpu --format=csv,noheader || true