File size: 1,748 Bytes
dbc69f3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | #!/usr/bin/env bash
set -eo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/conda_env.sh"
set -u
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "${ROOT}"
BASE_CONFIG="${BASE_CONFIG:-configs/grpo_llama32_3b_bf16.yaml}"
ACCELERATE_CONFIG="${ACCELERATE_CONFIG:-configs/accelerate_ddp_4gpu.yaml}"
NUM_PROCESSES="${NUM_PROCESSES:-4}"
MODEL_DIR="${MODEL_DIR:-}"
if [[ -z "${MODEL_DIR}" ]]; then
echo "Missing MODEL_DIR. Example:"
echo " MODEL_DIR=/path/to/checkpoint OUTPUT_PATH=out.json ./scripts/eval_math_level1_4gpu.sh"
exit 1
fi
OUTPUT_PATH="${OUTPUT_PATH:-artifacts/eval/math_level1/accuracy.json}"
MAX_SAMPLES="${MAX_SAMPLES:--1}"
BATCH_SIZE="${BATCH_SIZE:-4}"
MATH_SPLIT="${MATH_SPLIT:-test}"
SAVE_ROLLOUTS="${SAVE_ROLLOUTS:-0}"
ROLLOUTS_MAX_SAMPLES="${ROLLOUTS_MAX_SAMPLES:-50}"
ROLLOUTS_OUT_PATH="${ROLLOUTS_OUT_PATH:-artifacts/eval/math_level1/rollouts.jsonl}"
MAX_PROMPT_LENGTH="${MAX_PROMPT_LENGTH:-}"
MAX_COMPLETION_LENGTH="${MAX_COMPLETION_LENGTH:-}"
export PYTHONPATH="${ROOT}/src"
export BASE_CONFIG
export MODEL_DIR
export OUTPUT_PATH
export MAX_SAMPLES
export BATCH_SIZE
export MATH_SPLIT
export SAVE_ROLLOUTS
export ROLLOUTS_MAX_SAMPLES
export ROLLOUTS_OUT_PATH
if [[ -n "${MAX_PROMPT_LENGTH}" ]]; then export MAX_PROMPT_LENGTH; fi
if [[ -n "${MAX_COMPLETION_LENGTH}" ]]; then export MAX_COMPLETION_LENGTH; fi
export WANDB_MODE="${WANDB_MODE:-offline}"
export HF_HUB_OFFLINE="${HF_HUB_OFFLINE:-1}"
export HF_DATASETS_OFFLINE="${HF_DATASETS_OFFLINE:-1}"
export TRANSFORMERS_OFFLINE="${TRANSFORMERS_OFFLINE:-1}"
accelerate launch \
--config_file "${ACCELERATE_CONFIG}" \
--num_processes "${NUM_PROCESSES}" \
src/eval_math_level1.py
|