merged_models/eval.sh

#!/bin/bash

# Find all checkpoint directories and sort them numerically
for checkpoint in $(ls -d checkpoint-* | sort -t '-' -k2 -n); do
    echo "Evaluating $checkpoint..."
    
    lm_eval --model hf \
        --model_args pretrained=$checkpoint,parallelize=True,trust_remote_code=True \
        --tasks multimedqa \
        --device cuda:0 \
        --batch_size auto \
        --output_path "results" \
        --log_samples
        
    echo "Finished evaluating $checkpoint"
    echo "----------------------------------------"
done