| # Takes over the tail of teacher_window.sh after it was stopped mid-generation (2026-10-06: teacher.py | |
| # restarted with Qwen's native tool-call format). Generates until the original deadline into the | |
| # same file (teacher.py skips seeds already there), then does exactly what teacher_window.sh would: | |
| # stop vLLM, build synth_teacher, resume the main run with the decay forced. | |
| # TEACHER_END=<epoch> [TEACHER_KINDS=a,b,b,...] setsid nohup bash scripts/teacher_window_resume.sh >/dev/null 2>&1 < /dev/null & | |
| set -u | |
| cd "$(dirname "$0")/.." | |
| source env.sh | |
| RUN=$TA_DATA/runs/main | |
| L=$TA_DATA/logs/teacher_window.log | |
| log() { echo "[$(date '+%F %T')] $*" | tee -a "$L"; } | |
| OUT=${TEACHER_OUT:-$TA_DATA/teacher/qwen38_20261006.jsonl} | |
| END=${TEACHER_END:?set TEACHER_END to the epoch generation must stop at} | |
| TRAIN_CMD=($TA_PY scripts/train.py --size L --engram 1 --lr 3e-3 --tokens 1e10 --max_minutes 0 --decay_T 4096 | |
| --eval_every 250 --snapshot_tokens 5e8 --out "$RUN") | |
| H=$($TA_PY -c "import time;print(round(max(0.0,($END-time.time())/3600),3))") | |
| if curl -sf http://127.0.0.1:8000/health >/dev/null; then | |
| log "resume: teacher (native tool format) for $H h -> $OUT ${TEACHER_KINDS:+kinds=$TEACHER_KINDS}" | |
| # TEACHER_KINDS: comma list; repeating a kind upweights it (e.g. the harder aggregate/not_found) | |
| $TA_PY scripts/teacher.py --hours "$H" --concurrency 48 --out "$OUT" ${TEACHER_KINDS:+--kinds "$TEACHER_KINDS"} \ | |
| >> "$TA_DATA/logs/teacher.log" 2>&1 | |
| log "teacher done: $(tail -1 "$TA_DATA/logs/teacher.log" | cut -c1-300)" | |
| else | |
| log "resume: vLLM not healthy; skipping the rest of generation" | |
| fi | |
| docker rm -f tinyagent-teacher >/dev/null 2>&1 || true | |
| $TA_PY scripts/gen_synth.py --only teacher >> "$L" 2>&1 | |
| log "resuming main run with the decay forced now" | |
| touch "$RUN/DECAY" | |
| nohup "${TRAIN_CMD[@]}" >> "$TA_DATA/runs/main.out" 2>&1 & | |
| log "resumed (pid $!)" | |