evaluation/valid.sh

#!/usr/bin/env bash
set -euo pipefail

# 确保输出目录存在
mkdir -p /pfs/lichenyi/work/evaluation/predictions_in
mkdir -p /pfs/lichenyi/work/evaluation/predictions_ood

for i in 100 200 300 400 500 600 700 748; do
    echo "======= Running checkpoint-$i ======="

    model_path="/pfs/lichenyi/work/train_ddp/checkpoint-$i"

    ########################
    # 1) in-domain valid
    ########################
    data_path="/pfs/lichenyi/work/evaluation/valid_in.json"

    torchrun --nproc_per_node 4 simple_valid.py \
        --model_path "${model_path}" \
        --out_path /pfs/lichenyi/work/evaluation/predictions_in \
        --data_path "${data_path}" \
        --dtype bf16 \
        --use_system \
        --temperature 0.0

    python get_valid_score.py \
        --pred_path "/pfs/lichenyi/work/evaluation/predictions_in/predictions_checkpoint-${i}.json" \
        --out_path "/pfs/lichenyi/work/evaluation/valid_score_in_${i}.json"

    ########################
    # 2) ood valid
    ########################
    data_path="/pfs/lichenyi/work/evaluation/valid_ood.json"

    torchrun --nproc_per_node 4 simple_valid.py \
        --model_path "${model_path}" \
        --data_path "${data_path}" \
        --out_path /pfs/lichenyi/work/evaluation/predictions_ood \
        --dtype bf16 \
        --use_system \
        --temperature 0.0

    python get_valid_score.py \
        --pred_path "/pfs/lichenyi/work/evaluation/predictions_ood/predictions_checkpoint-${i}.json" \
        --out_path "/pfs/lichenyi/work/evaluation/valid_score_ood_${i}.json"

done