Buckets:
| # Whole run, in stage order. Each stage writes a done marker and is skipped on restart; generation and | |
| # training have deadlines (stop taking new work, let in-flight work finish). Start inside tmux. | |
| set -euo pipefail | |
| cd "$(dirname "$0")" | |
| source .venv/bin/activate | |
| MODEL=${MODEL:-Qwen/Qwen3.5-4B} | |
| TOTAL_H=${TOTAL_H:-8} | |
| GEN_MIN=${GEN_MIN:-165} | |
| TRAIN_MIN=${TRAIN_MIN:-75} | |
| ABL_MIN=${ABL_MIN:-30} | |
| BUCKET=${BUCKET:-hf://buckets/Mercity/Desicion-Model-Custom/run1} | |
| mkdir -p results logs markers | |
| [ -f markers/START ] || date +%s > markers/START | |
| START=$(cat markers/START) | |
| END=$(python -c "print(int($START + float('$TOTAL_H') * 3600))") | |
| log() { echo "{\"time\": \"$(date -u +%FT%TZ)\", \"event\": \"$1\", \"stage\": \"$2\", \"hours_left\": $(python -c "print(round(($END-$(date +%s))/3600,2))")}" | tee -a logs/timeline.jsonl; } | |
| sync_bucket() { # best effort; never blocks the run on a failed upload | |
| hf buckets sync results "$BUCKET/results" >> logs/sync.log 2>&1 || true | |
| hf buckets sync logs "$BUCKET/logs" >> logs/sync.log 2>&1 || true | |
| hf buckets sync data "$BUCKET/data" --exclude "raw/*" >> logs/sync.log 2>&1 || true | |
| for f in *.py run_all.sh preflight.sh README.md; do [ -f "$f" ] && hf buckets cp "$f" "$BUCKET/code/$f" >> logs/sync.log 2>&1 || true; done | |
| } | |
| stage() { # stage NAME COMMAND... | |
| local name=$1; shift | |
| if [ -f "markers/$name.done" ]; then log skip "$name"; return; fi | |
| log start "$name" | |
| "$@" 2>&1 | tee -a "logs/$name.log" | |
| touch "markers/$name.done" | |
| log done "$name" | |
| sync_bucket | |
| } | |
| [ -f results/pip_freeze.txt ] || { uv pip freeze > results/pip_freeze.txt; nvidia-smi > results/nvidia_smi.txt; } | |
| stage prep python -u prep_data.py --model "$MODEL" | |
| stage probe python -u gen_reasoning.py probe --model "$MODEL" --budget-min "$GEN_MIN" | |
| stage baseline python -u evaluate.py predict --model "$MODEL" --readout base --reason base --latency base | |
| GEN_DEADLINE=$(python -c "import time,os; print(min(time.time() + $GEN_MIN*60, $END - 3.8*3600))") | |
| stage generate python -u gen_reasoning.py generate --model "$MODEL" --deadline "$GEN_DEADLINE" | |
| stage train_joint python -u train.py --model "$MODEL" --mode joint --alpha 0.5 --out results/adapters/joint --budget-min "$TRAIN_MIN" | |
| stage merge_joint python -u merge_lora.py --model "$MODEL" --adapter results/adapters/joint --out checkpoints/joint_merged | |
| stage train_gold python -u train.py --model "$MODEL" --mode readout_only --alpha 1.0 --out results/adapters/gold_only --budget-min "$ABL_MIN" | |
| stage merge_gold python -u merge_lora.py --model "$MODEL" --adapter results/adapters/gold_only --out checkpoints/gold_only_merged | |
| stage train_votes python -u train.py --model "$MODEL" --mode readout_only --alpha 0.0 --out results/adapters/votes_only --budget-min "$ABL_MIN" | |
| stage merge_votes python -u merge_lora.py --model "$MODEL" --adapter results/adapters/votes_only --out checkpoints/votes_only_merged | |
| # trained models are served from merged checkpoints (vLLM ignores LoRA on Qwen3.5); one engine each | |
| stage eval_joint python -u evaluate.py predict --model checkpoints/joint_merged --readout joint --reason joint --latency joint | |
| stage eval_gold python -u evaluate.py predict --model checkpoints/gold_only_merged --readout gold_only | |
| stage eval_votes python -u evaluate.py predict --model checkpoints/votes_only_merged --readout votes_only | |
| stage metrics python -u evaluate.py metrics | |
| stage report python -u report.py | |
| log finished all | |
Xet Storage Details
- Size:
- 3.45 kB
- Xet hash:
- 541c07e1bc75b1a0a7920e0e533d7143bfb17b53b732ec47e3b39259f8aac464
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.