#!/usr/bin/env bash
# L15 diagnostic sweep: 4 arms on GPUs 1-4 + live plot watcher on CPU.
set -uo pipefail
cd /egr/research-slim/ghoshavr/reasoning-by-superposition-main
export WANDB_MODE=offline
TORCH=/egr/research-slim/ghoshavr/conda-envs/superposition/bin/torchrun
PY=/egr/research-slim/ghoshavr/conda-envs/superposition/bin/python
mkdir -p logs figs/diag_L15

# confirm warm start exists
if [[ ! -f ckpts/star-coconut-L15-bfs-stage0-warm/checkpoint_99 ]]; then
  echo "MISSING warm ckpt: ckpts/star-coconut-L15-bfs-stage0-warm/checkpoint_99"
  exit 1
fi

launch () {
  CUDA_VISIBLE_DEVICES="$1" setsid nohup "$TORCH" \
    --standalone --nnodes 1 --nproc_per_node 1 --master_port "$2" \
    run.py "args/$3.yaml" > "logs/$3.log" 2>&1 < /dev/null &
  echo "gpu$1  port$2  $3"
}

launch 1 29801 diag_L15_frontier_thr095
launch 2 29802 diag_L15_frontier_thr085
launch 3 29803 diag_L15_ce_thr050
launch 4 29804 diag_L15_frontier_nobt_095

# live plotter: refresh every 2 minutes
setsid nohup "$PY" scripts/plot_diag_L15.py --watch 120 \
  > logs/plot_diag_L15.log 2>&1 < /dev/null &
echo "plot watcher pid $!"

sleep 180
echo '=== procs ==='
pgrep -af 'run.py args/diag_L15_|plot_diag_L15' | grep -v pgrep || true
echo '=== GPUs ==='
nvidia-smi --query-gpu=index,memory.used,utilization.gpu --format=csv,noheader -i 1,2,3,4 || true
for f in diag_L15_frontier_thr095 diag_L15_frontier_thr085 diag_L15_ce_thr050 diag_L15_frontier_nobt_095; do
  echo "--- $f"
  grep -E 'acc-stage|train epoch|Traceback|Error|Missing' "logs/$f.log" 2>/dev/null | tail -5 || true
done