Lattice_Spike / scripts /queue_v6_mix.sh
Md Saiful Islam Sajol
Claude Opus 4.8
Sync to GitHub main fd62f0f: THRIVE v6.1 weights, v6/TriviaQA pipeline, labelling tooling
ac442f8
Raw History Blame Contribute Delete
1.2 kB
#!/usr/bin/env bash
# Overnight queue: wait for the rt_lr screen to finish, then run SET A
# (v6 objective-mix screen, 4 arms, one 4-wide wave).
#
# Start condition, whichever comes first:
# - all 10 rt_lr arms have written summary.json, or
# - no rt_lr run.log has been touched for 30 minutes (arms done or dead --
# either way the GPU is free, so don't waste the night waiting).
#
# tmux new-session -d -s v6mix 'bash scripts/queue_v6_mix.sh'
set -u
cd "$(dirname "${BASH_SOURCE[0]}")/.."
echo "[queue] waiting for rt_lr screen to finish $(date -Is)"
while true; do
n=$(ls outputs/rt_lr/lr*/summary.json 2>/dev/null | wc -l)
active=$(find outputs/rt_lr -name run.log -mmin -30 2>/dev/null | wc -l)
if [[ "$n" -ge 10 ]]; then
echo "[queue] all $n arms complete $(date -Is)"; break
fi
if [[ "$active" -eq 0 ]]; then
echo "[queue] no rt_lr activity for 30 min ($n/10 summaries) -- GPU idle, proceeding $(date -Is)"
break
fi
sleep 120
done
sleep 60 # let the screen's collector finish printing its table
echo "[queue] launching SET A $(date -Is)"
SETS=A bash scripts/run_v6_mix.sh 2>&1 | tee outputs/v6_mix.log
echo "[queue] SET A finished $(date -Is)"