launch/evaluation_hf.sh

#!/bin/bash

set -x

echo "WORKER_DIR: $WORKER_DIR"

LM_EVAL_REPO=${EVALUATION_REPO:-${WORKER_DIR}/lm-evaluation-harness}
# Change the path to your own
export PYTHONPATH=$PYTHONPATH:${WORKER_DIR}/Megatron-DeepSpeed

# ds_to_universal.py is part of deepspeed now. We can use it directly from deepspeed
DS_TO_UNIV_PY=${DS_TO_UNIV_PY:-/usr/local/lib/python3.10/dist-packages/deepspeed/checkpoint/ds_to_universal.py}

# change the path to your own checkpoint folder
ckpt_dir="/mnt/weka/peacock/peacock-data/experiments/llama/checkpoint/llamav2-3b/mbs8_240000/1024"
ckpt_step=(global_step120000 global_step240000)

# change the path to your own folder of the output, for universal and huggingface
UNIV_CKPT=$ckpt_dir/universal
HF_CKPT=$ckpt_dir/hf

# Use the scripts and json file for universal to huggingface
UNIV_TO_HF_PY="${WORKER_DIR}/convert_checkpoint/mds_universal_to_huggingface.py"
UNIV_TO_HF_JSON="${WORKER_DIR}/convert_checkpoint/mds_to_hf_llama_custom_3b_peacock.json"
tokenizer="${WORKER_DIR}/ConvertedTokenizer"

logs_eval=${HF_CKPT}/logs_eval
mkdir -p $logs_eval

tasks=(winogrande)

for ckpt in ${ckpt_step[@]}; do
	MDS_CKPT_DIR=$ckpt_dir/$ckpt
	UNIV_CKPT_DIR=$UNIV_CKPT/$ckpt
	HF_CKPT_DIR=$HF_CKPT/$ckpt

	mkdir -p $UNIV_CKPT_DIR
	mkdir -p $HF_CKPT_DIR

	# Convert to universal
	#python3 ${DS_TO_UNIV_PY} --input_folder ${MDS_CKPT_DIR} --output_folder ${UNIV_CKPT_DIR}

	# Convert to huggingface
	#python3 ${UNIV_TO_HF_PY} --universal-dir ${UNIV_CKPT_DIR} --output-dir ${HF_CKPT_DIR} --model-type llama --config ${UNIV_TO_HF_JSON}

	# Run evaluation now
	cd $LM_EVAL_REPO
	export HF_DATASETS_TRUST_REMOTE_CODE=True
	for task in ${tasks[@]}; do
		cmd="HF_DATASETS_TRUST_REMOTE_CODE=True lm_eval --model hf --model_args "pretrained=${HF_CKPT_DIR},tokenizer=${tokenizer}" --tasks $task --device hpu --batch_size 8   --num_fewshot 5 --limit 0.1 --trust_remote_code --verbosity DEBUG 2>&1 |tee ${logs_eval}/${ckpt}_${task}_bs8_0.1.log"
		echo $cmd
		eval $cmd
	done

done