Buckets:
| # Phase B after RL: merge one RL checkpoint into V2 joint, evaluate it exactly as V2 was (readout on all eval items, | |
| # thinking mode on the 1,498-item subset plus calibration, latency), then metrics with the RL model beside V2 and v1. | |
| # CKPT=checkpoints/rl_grpo/checkpoint-60 NAME=rl60 bash run_rl_eval.sh | |
| # Stage markers make it resumable; logs go to ../../logs/v2_<stage>.log and sync to the bucket like run_v2.sh. | |
| set -euo pipefail | |
| cd "$(dirname "$0")" | |
| source .venv/bin/activate | |
| CKPT=${CKPT:?set CKPT to an RL checkpoint dir} | |
| NAME=${NAME:-rl} | |
| BUCKET=${BUCKET:-hf://buckets/Mercity/Desicion-Model-Custom/run2_v2} | |
| L=../../logs | |
| mkdir -p results markers checkpoints | |
| log() { echo "{\"time\": \"$(date -u +%FT%TZ)\", \"event\": \"$1\", \"stage\": \"$2\"}" | tee -a $L/v2_timeline.jsonl; } | |
| sync_bucket() { | |
| hf buckets sync results "$BUCKET/results" --exclude "predictions/*.raw.jsonl" >> $L/v2_sync.log 2>&1 || true | |
| for f in $L/v2_*.log $L/v2_timeline.jsonl $L/rl_grpo.log; do [ -f "$f" ] && hf buckets cp "$f" "$BUCKET/logs/$(basename $f)" >> $L/v2_sync.log 2>&1 || true; done | |
| } | |
| stage() { local name=$1; shift; if [ -f "markers/$name.done" ]; then log skip "$name"; return; fi | |
| log start "$name"; "$@" 2>&1 | tee -a "$L/v2_$name.log"; touch "markers/$name.done"; log done "$name"; sync_bucket; } | |
| stage merge_$NAME python -u merge_lora.py --model checkpoints/v2_joint_merged --adapter "$CKPT" --out checkpoints/${NAME}_merged | |
| stage eval_$NAME env MAX_NUM_SEQS=160 GPU_MEM=0.92 python -u evaluate.py predict --model checkpoints/${NAME}_merged --readout $NAME \ | |
| --reason $NAME --latency $NAME --reason-subset data/eval_subset_ids.txt | |
| P0=../../results/predictions; P=results/predictions | |
| # every RL model evaluated so far goes into one comparison (rl60, then rl40 after it) | |
| RLR=""; RLS=""; RLP="" | |
| for n in $(ls $P | sed -n 's/^readout_\(rl[0-9a-z_]*\)\.jsonl$/\1/p'); do | |
| RLR="$RLR $n=$P/readout_$n.jsonl" | |
| if [ -f $P/reason_$n.jsonl ]; then RLS="$RLS $n=$P/reason_$n.jsonl"; RLP="$RLP $n=$n"; fi | |
| done | |
| stage metrics_$NAME python -u metrics_v2.py --readout base=$P0/readout_base.jsonl v1_gold=$P0/readout_gold_only.jsonl \ | |
| v1_joint=$P0/readout_joint.jsonl v2_joint=$P/readout_v2_joint.jsonl $RLR \ | |
| --reason base=$P0/reason_base.jsonl v1_joint=$P0/reason_joint.jsonl v2_joint=$P/reason_v2_joint.jsonl $RLS \ | |
| --partner base=base v1_joint=v1_joint v2_joint=v2_joint $RLP --out results/metrics_rl.json | |
| log finished eval_$NAME | |
Xet Storage Details
- Size:
- 2.45 kB
- Xet hash:
- 40f84fb74ff01d7ea06cca82e869ee15686fe357e312446b14836dd081cc9963
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.