Buckets:
| # The ~2B model on the V2 recipe (user request: start it overnight whatever the RL result). | |
| # probe Qwen3.5-2B and MiniCPM5-2B -> pick by data -> smoke test -> 3-rate sweep -> V2 joint -> merge -> eval -> metrics | |
| # Data are V2's (data -> ../v2/data): the 4B's and the 35B teacher's chains serve as distillation targets for the 2B | |
| # (no time for 2B-generated chains overnight). Stage markers make it resumable; logs go to ../../logs/v2b_<stage>.log, | |
| # events to ../../logs/v2_timeline.jsonl as "2b_<stage>", results sync to the bucket under run3_2b/. | |
| set -euo pipefail | |
| cd "$(dirname "$0")" | |
| source .venv/bin/activate | |
| BUCKET=${BUCKET:-hf://buckets/Mercity/Desicion-Model-Custom/run3_2b} | |
| L=../../logs | |
| mkdir -p results markers checkpoints | |
| log() { echo "{\"time\": \"$(date -u +%FT%TZ)\", \"event\": \"$1\", \"stage\": \"2b_$2\"}" | tee -a $L/v2_timeline.jsonl; } | |
| sync_bucket() { | |
| hf buckets sync results "$BUCKET/results" --exclude "predictions/*.raw.jsonl" >> $L/v2b_sync.log 2>&1 || true | |
| for f in $L/v2b_*.log; do [ -f "$f" ] && hf buckets cp "$f" "$BUCKET/logs/$(basename $f)" >> $L/v2b_sync.log 2>&1 || true; done | |
| } | |
| # returns the command's status itself, so a failure is never marked done (also when called after || or in an if) | |
| stage() { local name=$1; shift; if [ -f "markers/$name.done" ]; then log skip "$name"; return 0; fi | |
| log start "$name"; local rc=0; "$@" 2>&1 | tee -a "$L/v2b_$name.log" || rc=$? | |
| if [ $rc -ne 0 ]; then log failed "$name"; return $rc; fi | |
| touch "markers/$name.done"; log done "$name"; sync_bucket; } | |
| V2="--readout-pool all --min-correct 2 --second-chain-min 3 --rep-max 20 --drop-recall --strip-cw --post-soft --eval-order-frac 0.5 --dev data/dev.jsonl" | |
| SPEED="--micro-tokens 8192 --ckpt-above 8192" | |
| T="env PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True" | |
| # 1. probe both candidates the way the 4B candidates were probed (v1 eval set, first 128 pool items, 4 chains each) | |
| probe() { (cd ../.. && GPU_MEM=0.92 MAX_BATCHED_TOKENS=32768 python -u probe_models.py --model "$1" --out runs/v2_2b/results/probe); } | |
| stage probe_qwen probe Qwen/Qwen3.5-2B | |
| stage probe_minicpm probe openbmb/MiniCPM5-2B || log failed probe_minicpm # a MiniCPM tooling failure leaves Qwen | |
| # 2. pick (pre-registered): score = mean(readout accuracy over the 4 eval sets, first-chain accuracy on the pool); | |
| # a candidate below 0.9 parse rate is out; within 0.02 the same-family Qwen3.5-2B wins (tooling proven on the 4B) | |
| MODEL=$(python - <<'PYEOF' | |
| import json, os | |
| def score(p): | |
| r = json.load(open(p)); ro = sum(r["readout_acc"].values()) / len(r["readout_acc"]) | |
| pa = r["pool_acc"]; n = sum(v["n"] for v in pa.values()); t1 = sum(v["think1"] * v["n"] for v in pa.values()) / n | |
| return {"model": r["model"], "readout": ro, "think1": t1, "parse": r["think"]["parse_rate"], "looped": r["think"]["looped"], | |
| "tokens_median": r["think"]["tokens_median"], "score": (ro + t1) / 2} | |
| c = {k: score(f"results/probe/{f}.json") for k, f in (("qwen", "Qwen3.5-2B"), ("minicpm", "MiniCPM5-2B")) | |
| if os.path.exists(f"results/probe/{f}.json")} | |
| ok = {k: v for k, v in c.items() if v["parse"] >= 0.9} | |
| pick = "qwen" if ("qwen" in ok and (len(ok) == 1 or ok["qwen"]["score"] >= ok.get("minicpm", {"score": -1})["score"] - 0.02)) else ("minicpm" if "minicpm" in ok else "qwen") | |
| json.dump({"candidates": c, "picked": c[pick]["model"], "rule": "mean(readout over 4 sets, think1 on pool); parse >= 0.9; Qwen within 0.02"}, | |
| open("results/pick_2b.json", "w"), indent=1) | |
| print(c[pick]["model"]) | |
| PYEOF | |
| ) | |
| [ -f results/model_2b.txt ] && MODEL=$(cat results/model_2b.txt) # a resumed run keeps its model | |
| log chosen "model=$MODEL" | |
| # 3. smoke test: two optimizer steps on the sweep subset, so a model-specific failure costs minutes, not the night | |
| smoke() { $T python -u train.py --model "$1" --mode joint --alpha 0.5 --lr 2e-4 --subset data/sweep_ids.txt $V2 $SPEED \ | |
| --max-steps 2 --no-probe --out /root/bench_tmp/smoke_2b; } | |
| if ! stage smoke smoke "$MODEL"; then | |
| [ "$MODEL" = "Qwen/Qwen3.5-2B" ] && exit 1 | |
| MODEL=Qwen/Qwen3.5-2B; log chosen "model=$MODEL (fallback: the picked model failed the smoke test)" | |
| stage smoke_qwen smoke "$MODEL" | |
| fi | |
| # 4. short sweep, same rule as the 4B: the lowest rate within 0.01 dev NLL of the best | |
| echo "$MODEL" > results/model_2b.txt | |
| V2S=${V2/data\/dev.jsonl/data\/dev_sweep.jsonl} | |
| for LR in 1e-4 2e-4 3e-4; do | |
| stage sweep_$LR $T python -u train.py --model "$MODEL" --mode readout_only --alpha 0.5 --lr $LR --subset data/sweep_ids.txt \ | |
| $V2S $SPEED --no-probe --out results/adapters/sweep_$LR --budget-min 15 | |
| done | |
| BEST_LR=$(python - <<'PYEOF' | |
| import json | |
| r = {lr: json.load(open(f"results/adapters/sweep_{lr}/train_meta.json")) for lr in ("1e-4", "2e-4", "3e-4")} | |
| top = min(r[lr]["dev_nll"] for lr in r) | |
| best = min((lr for lr in r if r[lr]["dev_nll"] <= top + 0.01), key=float) | |
| json.dump({lr: {k: r[lr][k] for k in ("dev_acc", "dev_nll", "dev_ece15")} for lr in r} | {"best": best}, open("results/lr_sweep.json", "w"), indent=1) | |
| print(best) | |
| PYEOF | |
| ) | |
| log chosen "lr=$BEST_LR" | |
| # 5. V2 joint on the 2B, merge, evaluate like the 4B (base readout too, for the training gain) | |
| stage train_joint $T python -u train.py --model "$MODEL" --mode joint --alpha 0.5 --lr $BEST_LR $V2 $SPEED \ | |
| --out results/adapters/v2b_joint --budget-min ${TRAIN_MIN:-240} | |
| stage merge_joint python -u merge_lora.py --model "$MODEL" --adapter results/adapters/v2b_joint --out checkpoints/v2b_joint_merged | |
| stage eval_base python -u evaluate.py predict --model "$MODEL" --readout base2b | |
| stage eval_joint env MAX_NUM_SEQS=96 GPU_MEM=0.92 python -u evaluate.py predict --model checkpoints/v2b_joint_merged --readout v2b \ | |
| --reason v2b --latency v2b --reason-subset data/eval_subset_ids.txt | |
| P0=../../results/predictions; P4=../v2/results/predictions; P=results/predictions | |
| stage metrics python -u metrics_v2.py --readout base=$P0/readout_base.jsonl v1_joint=$P0/readout_joint.jsonl v2_joint=$P4/readout_v2_joint.jsonl \ | |
| base2b=$P/readout_base2b.jsonl v2b=$P/readout_v2b.jsonl \ | |
| --reason base=$P0/reason_base.jsonl v1_joint=$P0/reason_joint.jsonl v2_joint=$P4/reason_v2_joint.jsonl v2b=$P/reason_v2b.jsonl \ | |
| --partner base=base v1_joint=v1_joint v2_joint=v2_joint v2b=v2b --out results/metrics_2b.json | |
| log finished phase_2b | |
Xet Storage Details
- Size:
- 6.29 kB
- Xet hash:
- 779de51b7b50e30855d546f1cfe1a2d8d2d2516a962289f89969bfe6ab1fb508
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.