Buckets:
| # V2, Phase A, after generation: teacher merge -> speed benchmark -> short learning-rate sweep -> V2 joint | |
| # -> two small ablations -> evals -> metrics. | |
| # Stage markers make it resumable; every stage logs to ../../logs/v2_<stage>.log and syncs to the bucket. | |
| set -euo pipefail | |
| cd "$(dirname "$0")" | |
| source .venv/bin/activate | |
| MODEL=${MODEL:-Qwen/Qwen3.5-4B} | |
| BUCKET=${BUCKET:-hf://buckets/Mercity/Desicion-Model-Custom/run2_v2} | |
| L=../../logs | |
| mkdir -p results markers checkpoints | |
| log() { echo "{\"time\": \"$(date -u +%FT%TZ)\", \"event\": \"$1\", \"stage\": \"$2\"}" | tee -a $L/v2_timeline.jsonl; } | |
| sync_bucket() { | |
| hf buckets sync results "$BUCKET/results" --exclude "predictions/*.raw.jsonl" >> $L/v2_sync.log 2>&1 || true | |
| hf buckets sync data "$BUCKET/data" --exclude "raw/*" >> $L/v2_sync.log 2>&1 || true | |
| for f in $L/v2_*.log $L/v2_timeline.jsonl; do [ -f "$f" ] && hf buckets cp "$f" "$BUCKET/logs/$(basename $f)" >> $L/v2_sync.log 2>&1 || true; done | |
| } | |
| stage() { local name=$1; shift; if [ -f "markers/$name.done" ]; then log skip "$name"; return; fi | |
| log start "$name"; "$@" 2>&1 | tee -a "$L/v2_$name.log"; touch "markers/$name.done"; log done "$name"; sync_bucket; } | |
| V2="--readout-pool all --min-correct 2 --second-chain-min 3 --rep-max 20 --drop-recall --strip-cw --post-soft --eval-order-frac 0.5 --dev data/dev.jsonl" | |
| # teacher chains (OpenRouter, items from the back of the pool) join the local ones; local records win on a clash | |
| merge_teacher() { python - <<'EOF' | |
| import json, os | |
| if os.path.exists("data/gen_teacher.jsonl"): | |
| have = {json.loads(l)["id"] for l in open("data/gen.jsonl")} | |
| add = [l for l in open("data/gen_teacher.jsonl") if json.loads(l)["id"] not in have] | |
| with open("data/gen.jsonl", "a") as f: | |
| f.writelines(add) | |
| print(f"merged {len(add)} teacher records") | |
| EOF | |
| } | |
| stage merge_teacher merge_teacher | |
| # fixed subsets (seeded): 1,500 questions for the sweep (scored on 500 dev questions), 12,000 for the ablations | |
| # (all chain items first) | |
| [ -f data/sweep_ids.txt ] || python - <<'EOF' | |
| import json, random | |
| pool = [json.loads(l) for l in open("data/train_pool.jsonl")] | |
| gen = {json.loads(l)["id"] for l in open("data/gen.jsonl")} | |
| rng = random.Random(29) | |
| ids = [x["id"] for x in pool] | |
| rng.shuffle(ids) | |
| open("data/sweep_ids.txt", "w").write("\n".join(ids[:1500]) + "\n") | |
| chain = [i for i in ids if i in gen] | |
| rest = [i for i in ids if i not in gen] | |
| abl = (chain + rest)[:12000] | |
| open("data/ablation_ids.txt", "w").write("\n".join(abl) + "\n") | |
| dev = [json.loads(l) for l in open("data/dev.jsonl")] | |
| random.Random(31).shuffle(dev) | |
| with open("data/dev_sweep.jsonl", "w") as f: | |
| f.writelines(json.dumps(x, ensure_ascii=False) + "\n" for x in dev[:500]) | |
| print(f"sweep 1500; dev_sweep 500; ablation {len(abl)} ({min(len(chain), 12000)} with chains)") | |
| EOF | |
| # speed: the fastest setup that matches the current one numerically (no flags if none is worth it). Optional: if the | |
| # benchmark fails, training runs with the current setup instead of the pipeline stopping | |
| stage speed_bench python -u bench_speed.py --model "$MODEL" || log failed speed_bench | |
| BENCH=$(python -c "import json; print(json.load(open('results/speed_bench.json'))['flags'])" 2>/dev/null || true) | |
| # deviation #26: config C (+48%, numerically equivalent) taken over the benchmark's 72 GB line; its 74.9 GB peak is | |
| # 69.8 GiB of the card's 80 GiB. train.py redoes an unchecked micro-batch with checkpointing if it runs out of memory | |
| SPEED="--micro-tokens 8192 --ckpt-above 8192" | |
| T="env PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True" | |
| log chosen "speed=$SPEED (benchmark rule: ${BENCH:-default})" | |
| V2S=${V2/data\/dev.jsonl/data\/dev_sweep.jsonl} | |
| for LR in 1e-4 2e-4 3e-4; do | |
| stage sweep_$LR python -u train.py --model "$MODEL" --mode readout_only --alpha 0.5 --lr $LR --subset data/sweep_ids.txt \ | |
| $V2S $SPEED --no-probe --out results/adapters/sweep_$LR --budget-min 15 | |
| done | |
| BEST_LR=$(python - <<'EOF' | |
| import json | |
| r = {lr: json.load(open(f"results/adapters/sweep_{lr}/train_meta.json")) for lr in ("1e-4", "2e-4", "3e-4")} | |
| # pre-registered: the lowest rate whose dev NLL is within 0.01 of the best (a ~23-step run favours high rates) | |
| top = min(r[lr]["dev_nll"] for lr in r) | |
| best = min((lr for lr in r if r[lr]["dev_nll"] <= top + 0.01), key=float) | |
| json.dump({lr: {k: r[lr][k] for k in ("dev_acc", "dev_nll", "dev_ece15")} for lr in r} | {"best": best}, open("results/lr_sweep.json", "w"), indent=1) | |
| print(best) | |
| EOF | |
| ) | |
| log chosen "lr=$BEST_LR" | |
| stage train_v2_joint $T python -u train.py --model "$MODEL" --mode joint --alpha 0.5 --lr $BEST_LR $V2 $SPEED \ | |
| --out results/adapters/v2_joint --budget-min ${TRAIN_MIN:-300} | |
| stage merge_v2_joint python -u merge_lora.py --model "$MODEL" --adapter results/adapters/v2_joint --out checkpoints/v2_joint_merged | |
| stage train_abl_gold $T python -u train.py --model "$MODEL" --mode readout_only --alpha 1.0 --lr $BEST_LR --subset data/ablation_ids.txt \ | |
| $V2 $SPEED --out results/adapters/abl_gold --budget-min 45 | |
| stage train_abl_mix $T python -u train.py --model "$MODEL" --mode readout_only --alpha 0.5 --lr $BEST_LR --subset data/ablation_ids.txt \ | |
| $V2 $SPEED --out results/adapters/abl_mix --budget-min 45 | |
| stage merge_abl_gold python -u merge_lora.py --model "$MODEL" --adapter results/adapters/abl_gold --out checkpoints/abl_gold_merged | |
| stage merge_abl_mix python -u merge_lora.py --model "$MODEL" --adapter results/adapters/abl_mix --out checkpoints/abl_mix_merged | |
| stage eval_v2_joint env MAX_NUM_SEQS=128 python -u evaluate.py predict --model checkpoints/v2_joint_merged --readout v2_joint \ | |
| --reason v2_joint --latency v2_joint --reason-subset data/eval_subset_ids.txt | |
| stage eval_abl_gold python -u evaluate.py predict --model checkpoints/abl_gold_merged --readout abl_gold | |
| stage eval_abl_mix python -u evaluate.py predict --model checkpoints/abl_mix_merged --readout abl_mix | |
| P0=../../results/predictions; P=results/predictions | |
| stage metrics_v2 python -u metrics_v2.py --readout base=$P0/readout_base.jsonl v1_gold=$P0/readout_gold_only.jsonl v1_joint=$P0/readout_joint.jsonl \ | |
| v2_joint=$P/readout_v2_joint.jsonl abl_gold=$P/readout_abl_gold.jsonl abl_mix=$P/readout_abl_mix.jsonl \ | |
| --reason base=$P0/reason_base.jsonl v1_joint=$P0/reason_joint.jsonl v2_joint=$P/reason_v2_joint.jsonl \ | |
| --partner base=base v1_joint=v1_joint v2_joint=v2_joint --out results/metrics_v2.json | |
| log finished phase_a | |
Xet Storage Details
- Size:
- 6.38 kB
- Xet hash:
- e1a555c886bfdc5b6ccf84200d229a98bd8a39cacd14ae7a291b65a218e49299
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.