File size: 1,476 Bytes
b2f3bf4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | #!/usr/bin/env bash
# Mid-training review gate: pause the trainer, calibrate + evaluate a checkpoint, resume.
# usage: scripts/review_checkpoint.sh <ckpt_dir> <tag> [trainer_pid] [perm_rows]
set -euo pipefail
cd "$(dirname "$0")/.."
CKPT="$1"; TAG="$2"; PID="${3:-}"; PERM="${4:-500}"
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
mkdir -p reports/review logs
resume() { if [ -n "$PID" ]; then kill -CONT "$PID" 2>/dev/null && echo "resumed trainer $PID"; fi; }
trap resume EXIT
if [ -n "$PID" ]; then kill -STOP "$PID" && echo "paused trainer $PID"; fi
# snapshot the checkpoint so the trainer cannot overwrite it mid-review
SNAP="reports/review/${TAG}_ckpt"; rm -rf "$SNAP"; cp -r "$CKPT" "$SNAP"
python3 -c "import json;print(json.load(open('$SNAP/train_state.json')))"
python3 scripts/fit_temperature.py --checkpoint "$SNAP" --out "$SNAP/calibration.json" 2>&1 | grep -v -iE "warn|deprecat|USDT|it/s\]" | grep -E "rows|\"(noul|choice|score)\"|kl|ece|->" || true
python3 scripts/evaluate.py --checkpoint "$SNAP" --temperature "$SNAP/calibration.json" \
--splits test_set_30k,ood --out "reports/review/${TAG}.md" --json "reports/review/${TAG}.json" \
--baseline-json reports/b0_qwen35_9b.json --perm-rows "$PERM" 2>&1 | grep -v -iE "warn|deprecat|USDT|it/s\]" | grep -E "^\[|report" || true
rm -rf "$SNAP"/model-*.safetensors 2>/dev/null || true
echo "=== acceptance (excl. D1) ==="
sed -n '/excluding yuri_v1/,/^$/p' "reports/review/${TAG}.md" | head -12
|