#!/bin/bash
# supervise21.sh - keep the v21 run going until TARGET tokens are reached.
# train_peer_v21.py resumes from ckpt-peer21/resume.pt, so a relaunch loses no tokens.
cd /root/dna
TARGET=${TARGET:-1553000000}     # match the v20 baseline's final token count
while true; do
  if pgrep -f train_peer_v21.py >/dev/null; then sleep 120; continue; fi
  TOK=$(grep -o 'tokens=[0-9]*' peer21.log 2>/dev/null | tail -1 | cut -d= -f2)
  TOK=${TOK:-0}
  if [ "$TOK" -ge "$TARGET" ]; then
    echo "supervisor: TARGET reached ($TOK >= $TARGET) $(date)" >> supervisor.log
    break
  fi
  echo "supervisor: relaunching at $TOK tokens $(date)" >> supervisor.log
  MAXH=24 setsid nohup ./run21.sh >> peer21.log 2>&1 < /dev/null &
  sleep 180
done