Buckets:
| # After the 2B joint eval: (1) the untrained 2B's thinking mode on the same 1,498 items (+ calibration), so the joint | |
| # model's thinking can be compared with what it started from; (2) only if V2-joint training lowered the 2B's thinking | |
| # accuracy by more than 1 point: a readout-only 2B on the same 42K questions (no chain training, so the model's own | |
| # thinking is touched only through the shared LoRA), merged and evaluated the same way. | |
| set -uo pipefail | |
| cd "$(dirname "$0")" | |
| source .venv/bin/activate | |
| L=../../logs | |
| M=$(cat results/model_2b.txt) | |
| log() { echo "{\"time\": \"$(date -u +%FT%TZ)\", \"event\": \"$1\", \"stage\": \"2b_$2\"}" >> $L/v2_timeline.jsonl; } | |
| V2="--readout-pool all --min-correct 2 --second-chain-min 3 --rep-max 20 --drop-recall --strip-cw --post-soft --eval-order-frac 0.5 --dev data/dev.jsonl" | |
| if [ ! -f results/predictions/reason_base2b.jsonl ]; then | |
| log start eval_base_think | |
| env MAX_NUM_SEQS=96 GPU_MEM=0.92 python -u evaluate.py predict --model "$M" --reason base2b --reason-subset data/eval_subset_ids.txt > $L/v2b_eval_base_think.log 2>&1 \ | |
| && log done eval_base_think || log failed eval_base_think | |
| fi | |
| DROP=$(python - <<'PYEOF' | |
| import json | |
| ev = {json.loads(l)["id"]: json.loads(l) for l in open("data/eval.jsonl")} | |
| def acc(p): | |
| r = [json.loads(l) for l in open(p)] | |
| r = [x for x in r if x["id"] in ev] | |
| return {x["id"]: (x["answer"] is not None and x["answer"] == ev[x["id"]]["gold"]) for x in r} | |
| try: | |
| a, b = acc("results/predictions/reason_base2b.jsonl"), acc("results/predictions/reason_v2b.jsonl") | |
| ids = [i for i in a if i in b] | |
| print(round(100 * (sum(a[i] for i in ids) - sum(b[i] for i in ids)) / len(ids), 2)) | |
| except Exception: | |
| print("nan") | |
| PYEOF | |
| ) | |
| echo "{\"thinking_drop_points\": \"$DROP\"}" > results/joint_vs_base_thinking.json | |
| log chosen "thinking_drop=$DROP" | |
| if python -c "import sys; d='$DROP'; sys.exit(0 if d != 'nan' and float(d) > 1.0 else 1)"; then | |
| log start train_ro | |
| env PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True python -u train.py --model "$M" --mode readout_only --alpha 0.5 --lr 2e-4 $V2 \ | |
| --micro-tokens 8192 --ckpt-above 8192 --out results/adapters/v2b_ro --budget-min 90 > $L/v2b_train_ro.log 2>&1 && log done train_ro || { log failed train_ro; exit 0; } | |
| python -u merge_lora.py --model "$M" --adapter results/adapters/v2b_ro --out checkpoints/v2b_ro_merged > $L/v2b_merge_ro.log 2>&1 || { log failed merge_ro; exit 0; } | |
| log start eval_ro | |
| env MAX_NUM_SEQS=96 GPU_MEM=0.92 python -u evaluate.py predict --model checkpoints/v2b_ro_merged --readout v2b_ro --reason v2b_ro \ | |
| --reason-subset data/eval_subset_ids.txt > $L/v2b_eval_ro.log 2>&1 && log done eval_ro || log failed eval_ro | |
| fi | |
Xet Storage Details
- Size:
- 2.72 kB
- Xet hash:
- 68b8fd8d4f121c8c85c0493e88376868d660ee280e2e0bcb8dce4bdeb52af5de
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.