File size: 740 Bytes
134783f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | #!/bin/bash
# supervise21.sh - keep the v21 run going until TARGET tokens are reached.
# train_peer_v21.py resumes from ckpt-peer21/resume.pt, so a relaunch loses no tokens.
cd /root/dna
TARGET=${TARGET:-1553000000} # match the v20 baseline's final token count
while true; do
if pgrep -f train_peer_v21.py >/dev/null; then sleep 120; continue; fi
TOK=$(grep -o 'tokens=[0-9]*' peer21.log 2>/dev/null | tail -1 | cut -d= -f2)
TOK=${TOK:-0}
if [ "$TOK" -ge "$TARGET" ]; then
echo "supervisor: TARGET reached ($TOK >= $TARGET) $(date)" >> supervisor.log
break
fi
echo "supervisor: relaunching at $TOK tokens $(date)" >> supervisor.log
MAXH=24 setsid nohup ./run21.sh >> peer21.log 2>&1 < /dev/null &
sleep 180
done
|