#!/usr/bin/env bash # Overnight queue: wait for the rt_lr screen to finish, then run SET A # (v6 objective-mix screen, 4 arms, one 4-wide wave). # # Start condition, whichever comes first: # - all 10 rt_lr arms have written summary.json, or # - no rt_lr run.log has been touched for 30 minutes (arms done or dead -- # either way the GPU is free, so don't waste the night waiting). # # tmux new-session -d -s v6mix 'bash scripts/queue_v6_mix.sh' set -u cd "$(dirname "${BASH_SOURCE[0]}")/.." echo "[queue] waiting for rt_lr screen to finish $(date -Is)" while true; do n=$(ls outputs/rt_lr/lr*/summary.json 2>/dev/null | wc -l) active=$(find outputs/rt_lr -name run.log -mmin -30 2>/dev/null | wc -l) if [[ "$n" -ge 10 ]]; then echo "[queue] all $n arms complete $(date -Is)"; break fi if [[ "$active" -eq 0 ]]; then echo "[queue] no rt_lr activity for 30 min ($n/10 summaries) -- GPU idle, proceeding $(date -Is)" break fi sleep 120 done sleep 60 # let the screen's collector finish printing its table echo "[queue] launching SET A $(date -Is)" SETS=A bash scripts/run_v6_mix.sh 2>&1 | tee outputs/v6_mix.log echo "[queue] SET A finished $(date -Is)"