Spaces:
Running
Running
File size: 3,750 Bytes
cf6c0e0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | #!/usr/bin/env bash
# Run after training completes to evaluate trained LoRA vs base and start run 2.
# Usage: bash scripts/eval_after_training.sh [lora_dir]
#
# Steps:
# 1. Start vLLM with base model (qwen35) + trained LoRA (qwen35-trained)
# 2. Run inference.py with base model → measure baseline
# 3. Run inference.py with trained model → measure improvement
# 4. Print comparison and write to checkpoints/eval_results.txt
# 5. Prompt whether to start run 2 (resume from trained LoRA)
set -euo pipefail
LORA_DIR="${1:-checkpoints/run2/developer_final}"
MODEL_PATH="$HOME/models/Qwen3.5-2B"
VLLM_LOG="$HOME/vllm_eval.log"
RESULTS_FILE="checkpoints/eval_results.txt"
PYTHON="/dev/shm/qwen35/bin/python"
echo "=== Post-Training Evaluation ==="
echo "LoRA: $LORA_DIR"
echo "Model: $MODEL_PATH"
echo ""
# Verify LoRA exists
if [ ! -d "$LORA_DIR" ]; then
echo "ERROR: LoRA dir not found: $LORA_DIR"
exit 1
fi
# Kill any lingering vLLM processes
if pgrep -f "vllm.entrypoints" > /dev/null 2>&1; then
echo "Killing existing vLLM process..."
pkill -f "vllm.entrypoints" || true
sleep 5
fi
echo "Starting vLLM with base + trained LoRA (tensor-parallel=2, port 8001)..."
apptainer exec --nv ~/apptainer-images/cuda-custom-amal_latest.sif bash -c "
export LD_PRELOAD=/dev/shm/qwen35/lib/libstdc++.so.6
/dev/shm/qwen35/bin/python -m vllm.entrypoints.openai.api_server \
--model $MODEL_PATH \
--served-model-name qwen35 \
--tensor-parallel-size 2 \
--port 8001 \
--host 0.0.0.0 \
--max-model-len 65536 \
--enable-auto-tool-choice \
--tool-call-parser hermes \
--enable-lora \
--lora-modules qwen35-trained=$LORA_DIR
" 2>&1 | tee "$VLLM_LOG" &
VLLM_PID=$!
echo "Waiting for vLLM to be ready..."
for i in $(seq 1 120); do
if curl -s http://localhost:8001/health > /dev/null 2>&1; then
echo "vLLM ready (${i}s)"
break
fi
sleep 5
done
echo ""
echo "--- BASELINE (base model, no LoRA) ---"
export API_BASE_URL="http://localhost:8001/v1"
export MODEL_NAME="qwen35"
export HF_TOKEN="sk-local"
export MAX_STEPS="3"
export INFERENCE_SERVER_PORT="18082"
$PYTHON inference.py 2>&1 | tee /tmp/eval_base.log
BASE_SCORE=$(grep "\[END\]" /tmp/eval_base.log | grep -oP "score=\K[\d.]+" | paste -sd+ | bc -l 2>/dev/null || echo "0")
echo ""
echo "--- TRAINED (with LoRA) ---"
export MODEL_NAME="qwen35-trained"
$PYTHON inference.py 2>&1 | tee /tmp/eval_trained.log
TRAINED_SCORE=$(grep "\[END\]" /tmp/eval_trained.log | grep -oP "score=\K[\d.]+" | paste -sd+ | bc -l 2>/dev/null || echo "0")
echo ""
echo "=== COMPARISON ==="
echo "Base mean: $BASE_SCORE"
echo "Trained mean: $TRAINED_SCORE"
{
echo "=== Evaluation Results $(date) ==="
echo "LoRA: $LORA_DIR"
echo ""
echo "Base inference log:"
grep "\[END\]\|\[START\]" /tmp/eval_base.log
echo ""
echo "Trained inference log:"
grep "\[END\]\|\[START\]" /tmp/eval_trained.log
} > "$RESULTS_FILE"
echo "Results saved to $RESULTS_FILE"
# Kill vLLM
kill $VLLM_PID 2>/dev/null || pkill -f "vllm.entrypoints" || true
wait $VLLM_PID 2>/dev/null || true
echo ""
echo "To start run 2 (resume from trained LoRA with fixed CRITIC + MAX_NEW_TOKENS=2048):"
echo " apptainer exec --nv ~/apptainer-images/cuda-custom-amal_latest.sif bash -c \\"
echo " 'export LD_PRELOAD=/dev/shm/qwen35/lib/libstdc++.so.6; \\"
echo " /dev/shm/qwen35/bin/python train.py \\"
echo " --phase developer \\"
echo " --episodes 10 \\"
echo " --k-rollouts 4 \\"
echo " --model $MODEL_PATH \\"
echo " --checkpoint-dir checkpoints/run3 \\"
echo " --resume-from $LORA_DIR' \\"
echo " 2>&1 | tee checkpoints/train_run3.log"
|