Buckets:
| # Third queue: after the 2-epoch queue (pid $1) exits, run Qwen3.5-0.8B for 1.5 then 2 epochs. | |
| # Uses the separate transformers-5 environment (.venv-qwen35); the system env stays on 4.57.3. | |
| # Each run: smoke test (writes resolved.yaml) -> training -> checkpoint/report upload to W&B. | |
| set -uo pipefail | |
| cd "$(dirname "$0")/.." | |
| export CUDA_VISIBLE_DEVICES=0 | |
| PY=.venv-qwen35/bin/python | |
| WAIT_PID=${1:-} | |
| if [[ -n "$WAIT_PID" ]]; then | |
| echo "[queue3] waiting for pid $WAIT_PID" | |
| while kill -0 "$WAIT_PID" 2>/dev/null; do sleep 20; done | |
| fi | |
| run_one() { # $1 config, $2 output dir, $3 log name | |
| echo "[queue3] $3 smoke test" | |
| if ! $PY -m training.smoke --config "$1" > "smoke_$3.log" 2>&1; then | |
| echo "[queue3] $3 smoke FAILED; see smoke_$3.log"; return 1 | |
| fi | |
| echo "[queue3] $3 smoke ok: $($PY -c "import json;s=json.load(open('$2/smoke_summary.json'));print(s['selected'], 'est train s', round(s['conservative_train_seconds']))")" | |
| echo "[queue3] $3 training" | |
| $PY -m training.train_qwen35 --config "$2/resolved.yaml" > "$3.log" 2>&1 | |
| echo "[queue3] $3 exit $?" | |
| if [[ -f "$2/final/adapter_config.json" ]]; then | |
| $PY scripts/log_checkpoints_wandb.py "$2" || echo "[queue3] $3 artifact upload FAILED" | |
| fi | |
| } | |
| # Stop after a failed smoke test: the 2-epoch run would fail the same way. | |
| run_one training/qwen35_08b_30k.yaml checkpoints/qwen3.5-0.8b-30k qwen35 && \ | |
| run_one training/qwen35_08b_30k_2ep.yaml checkpoints/qwen3.5-0.8b-30k-2ep qwen35_2ep | |
| echo "[queue3] done" | |
Xet Storage Details
- Size:
- 1.5 kB
- Xet hash:
- 485d92e6f2de42e7c4c62d2f02978a928242352467504bbaf58f45260720007b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.