File size: 2,186 Bytes
40e76c0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 | #!/bin/bash
# Combined perplexity + zero-shot eval for one checkpoint. Just point it at a model.
#
# eval_model.sh <model_path_or_hf_id> [name] [gpu]
#
# Runs, on one GPU, sequentially (PPL via HF, then zero-shot via lm_eval+vllm):
# - WT2 + C4 perplexity
# - 7-task zero-shot (arc_easy, arc_challenge, hellaswag, winogrande, piqa,
# boolq, openbookqa)
# then prints a combined table. Results go under ~/results/eval_<name>/.
# Zero-shot table uses acc_norm for arc/hellaswag/piqa/openbookqa, acc otherwise.
set -uo pipefail
. ~/local/venvs/main/bin/activate
HERE="$(cd "$(dirname "$0")" && pwd)"
MODEL="${1:?usage: eval_model.sh <model> [name] [gpu]}"
NAME="${2:-$(basename "$MODEL")}"
GPU="${3:-0}"
TASKS=arc_easy,arc_challenge,hellaswag,winogrande,piqa,boolq,openbookqa
OUT="$HOME/results/eval_${NAME}"
mkdir -p "$OUT"
echo "[eval] model=$MODEL name=$NAME gpu=$GPU -> $OUT"
echo "[eval] 1/2 perplexity (wikitext2 + c4)..."
CUDA_VISIBLE_DEVICES=$GPU python "$HERE/ppl_eval.py" "$MODEL" --out "$OUT/ppl.json"
echo "[eval] 2/2 zero-shot (lm_eval + vllm)..."
rm -rf "$OUT/zeroshot"
CUDA_VISIBLE_DEVICES=$GPU lm_eval --model vllm \
--model_args "pretrained=$MODEL,tensor_parallel_size=1,gpu_memory_utilization=0.85,dtype=bfloat16,max_model_len=4096,trust_remote_code=True" \
--tasks "$TASKS" --batch_size auto --output_path "$OUT/zeroshot"
echo "[eval] ===== $NAME ====="
python - "$OUT" <<'PY'
import glob, json, os, sys
OUT = sys.argv[1]
NORM = {"arc_easy", "arc_challenge", "hellaswag", "piqa", "openbookqa"}
TASKS = ["arc_easy", "arc_challenge", "hellaswag", "winogrande", "piqa", "boolq", "openbookqa"]
ppl = json.load(open(os.path.join(OUT, "ppl.json")))
print(f" WT2 {ppl['wikitext2']:.2f}")
print(f" C4 {ppl['c4']:.2f}")
fs = glob.glob(os.path.join(OUT, "zeroshot", "**", "*.json"), recursive=True)
r = (json.load(open(sorted(fs)[-1])).get("results", {})) if fs else {}
vals = {}
for t in TASKS:
x = r.get(t)
if x:
m = "acc_norm,none" if t in NORM else "acc,none"
vals[t] = x.get(m, x.get("acc,none")) * 100
print(f" {t:14s} {vals[t]:.2f}")
if vals:
print(f" {'ZS avg':14s} {sum(vals.values())/len(vals):.2f}")
PY
|