| # Run a list of train_qa.py argument strings, at most JOBS at a time. | |
| # | |
| # The GPU on this box is shared with other work and had ~2 GB free; launching | |
| # ten arms at once produced CUDA_STATUS_ALLOC_FAILED on four of them, which is | |
| # a silent way to lose a cell of a factorial. A queue is cheaper than | |
| # re-reading logs to find out which arms never ran. | |
| # | |
| # echo "--nexp 1 --seed 1 | |
| # --nexp 4 --seed 1" | train/qa_queue.sh | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| JOBS=${JOBS:-3} | |
| STEPS=${STEPS:-8000} | |
| OUT=${OUT:-runs/qa} | |
| mkdir -p "$OUT" runs/logs | |
| while IFS= read -r args; do | |
| [ -z "${args// /}" ] && continue | |
| while [ "$(jobs -rp | wc -l)" -ge "$JOBS" ]; do sleep 5; done | |
| tag=$(echo "$args" | tr -cd 'A-Za-z0-9.' | tr -s ' ' '_') | |
| echo "launch: $args" | |
| python3 train/train_qa.py --steps "$STEPS" --out "$OUT" $args \ | |
| > "runs/logs/$tag.log" 2>&1 & | |
| done | |
| wait | |
| echo "queue done" | |