File size: 3,196 Bytes
fb5235a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | #!/bin/bash
# Independent orchestrator: when the final QA finishes and the complete dataset
# is good, assemble it and switch training onto it WASTING NO TIME -- the kill is
# timed to fire immediately after a checkpoint save, so the relaunch resumes from
# the freshest weights. The always-on supervisor performs the actual relaunch
# (it already reads data/.../aligned6_* which we overwrite here with the complete
# set), so we only need to assemble + time the kill.
set -u
cd /Users/moncif/gemma4-hack
SW=runs/switch.log
log(){ echo "$(date '+%F %T'): $*" >> "$SW"; }
EMB=data/embeddings/aligned6_embeddings.parquet
QNA=data/qna/aligned6_qna.jsonl
TRAIN_PAT="train_memory_lora.py --output-dir sixview_v2"
log "orchestrator started"
# 1) wait for the final QA generation to finish
while pgrep -f "generate_repo_scoped_qa" >/dev/null 2>&1; do sleep 15; done
log "final QA finished"
# 2) report coverage
./venv/bin/python - >> "$SW" 2>&1 <<'PY'
import json
srcs={json.loads(l)['doc_id'] for l in open('data/docs/multiview_sources.jsonl')}
qa={json.loads(l)['doc_id'] for l in open('data/qna/repo_scoped_qa.jsonl')}
print(f"coverage: emb={len(srcs)} qa={len(qa)} missing={len(srcs-qa)}")
PY
# 3) assemble the COMPLETE aligned dataset (overwrites aligned6_* in place;
# the running trainer already holds its data in memory, so this is safe)
log "assembling complete aligned dataset ..."
./venv/bin/python scripts/assemble_6view_dataset.py >> "$SW" 2>&1
# 4) verify the assembled dataset is bigger/good before switching
REPOS=$(./venv/bin/python -c "import pyarrow.parquet as pq;print(pq.read_table('$EMB').num_rows)" 2>/dev/null || echo 0)
QCOUNT=$(wc -l < "$QNA" 2>/dev/null | tr -d ' ')
log "assembled: repos=$REPOS qa=$QCOUNT (was 1058 repos / 8540 qa)"
if [ "${REPOS:-0}" -lt 1400 ]; then
log "ABORT SWITCH: assembled repos ($REPOS) < 1400 -- keeping current dataset, NOT switching."
exit 1
fi
# 5) TIMED KILL: wait for the next checkpoint save (head.latest.pt mtime bump),
# then kill immediately so the resume loses ~0 steps.
log "waiting for next checkpoint save to time the kill (zero wasted steps) ..."
m0=$(stat -f %m "runs/sixview_v2/head.latest.pt" 2>/dev/null || echo 0)
killed=0
for i in $(seq 1 900); do # up to ~15 min safety
m1=$(stat -f %m "runs/sixview_v2/head.latest.pt" 2>/dev/null || echo 0)
if [ "$m1" != "$m0" ] && [ "$m1" != "0" ]; then
step=$(grep -oE "step[0-9]+" runs/sixview_v2_train.log | tail -1)
log "checkpoint just saved (mtime bumped) at $step; KILLING training now to switch dataset"
pkill -f "$TRAIN_PAT"
killed=1
break
fi
sleep 1
done
if [ "$killed" = 0 ]; then
log "no checkpoint save seen in 15min; killing anyway (latest.pt is still recent)"
pkill -f "$TRAIN_PAT"
fi
log "training killed; supervisor will relaunch on COMPLETE dataset ($REPOS repos) from head.latest.pt within ~45s"
# 6) confirm the relaunch actually happened on the new data
sleep 90
if pgrep -f "$TRAIN_PAT" >/dev/null 2>&1; then
log "SWITCH OK: training is running again (complete dataset, $REPOS repos)"
else
log "WARN: training not detected 90s after kill -- supervisor should relaunch; will self-heal"
fi
log "orchestrator done"
|