diff --git a/lm-evaluation-harness/tests/testdata/wmt20-en-zh-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-en-zh-v0-greedy_until
new file mode 100644
index 0000000000000000000000000000000000000000..db79b7f03fcfc8f7720f1344339e7d94d8a01ebf
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-en-zh-v0-greedy_until
@@ -0,0 +1 @@
+67f0333ddbcb07d7a9ac12919129a18fe4fea24e4826a11bbdde4fd5ed5ed83f
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-en-zh-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-en-zh-v0-res.json
new file mode 100644
index 0000000000000000000000000000000000000000..24db35e62fd176b0454ff426ab749787da805897
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-en-zh-v0-res.json
@@ -0,0 +1 @@
+{"results": {"wmt20-en-zh": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.00014170297316825535, "chrf_stderr": 6.590669847391838e-05, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-en-zh": 0}}
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-fr-de-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-fr-de-v0-res.json
new file mode 100644
index 0000000000000000000000000000000000000000..d5d06a02a30635ad57907b32ae66ccb9ba5a7e23
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-fr-de-v0-res.json
@@ -0,0 +1 @@
+{"results": {"wmt20-fr-de": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.01143193767396364, "chrf_stderr": 0.00012555271954563658, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-fr-de": 0}}
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-iu-en-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-iu-en-v0-greedy_until
new file mode 100644
index 0000000000000000000000000000000000000000..87a1981e79e0558366487aa2476cc751ad3857dc
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-iu-en-v0-greedy_until
@@ -0,0 +1 @@
+97bf664a8efa54b5366b8341f77b418106dd0cb26169d5b2d0144e4d3d2bc5c9
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-iu-en-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-iu-en-v0-res.json
new file mode 100644
index 0000000000000000000000000000000000000000..e94cac8876d9d04c883b5ad5810884af7faa436c
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-iu-en-v0-res.json
@@ -0,0 +1 @@
+{"results": {"wmt20-iu-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.012204628007572778, "chrf_stderr": 8.944407532175802e-05, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-iu-en": 0}}
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-ja-en-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-ja-en-v0-greedy_until
new file mode 100644
index 0000000000000000000000000000000000000000..3a89d7fcdfb76bc3912a930cf592da0270ba440c
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-ja-en-v0-greedy_until
@@ -0,0 +1 @@
+1fd846f3c0104e794eb380dae7f648592092ab8bf59234c26d0a671bbbc28df1
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-ja-en-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-ja-en-v0-res.json
new file mode 100644
index 0000000000000000000000000000000000000000..4344b7cd8a1a9bfb8cd60e2aa0ece17f530f7d3d
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-ja-en-v0-res.json
@@ -0,0 +1 @@
+{"results": {"wmt20-ja-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.010703148854351403, "chrf_stderr": 0.00022242113108130186, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-ja-en": 0}}
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-km-en-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-km-en-v0-greedy_until
new file mode 100644
index 0000000000000000000000000000000000000000..a6f148661093f5f97c85e96ab2c01da6e40c1547
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-km-en-v0-greedy_until
@@ -0,0 +1 @@
+fb4ec81bb89c70df7e21b43e0e882915b7b71a2a85bb8d4b59e0c7938baaa4c2
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-km-en-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-km-en-v0-res.json
new file mode 100644
index 0000000000000000000000000000000000000000..4f6dc98604bdeed7b87806094a6ffc3b0cbbfec4
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-km-en-v0-res.json
@@ -0,0 +1 @@
+{"results": {"wmt20-km-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.015142474534585969, "chrf_stderr": 0.0001518735048829897, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-km-en": 0}}
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-pl-en-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-pl-en-v0-greedy_until
new file mode 100644
index 0000000000000000000000000000000000000000..899ce01919910ab80fc95b99371bf334a7c7c37a
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-pl-en-v0-greedy_until
@@ -0,0 +1 @@
+89274499d84176b1ffe4eaec06f2c89ca807342384dc946c2e348d00116aaade
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-pl-en-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-pl-en-v0-res.json
new file mode 100644
index 0000000000000000000000000000000000000000..a2f5cb31be388a5ca081567e72d16e42774aa008
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-pl-en-v0-res.json
@@ -0,0 +1 @@
+{"results": {"wmt20-pl-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.01353367757716276, "chrf_stderr": 0.00018386199249976465, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-pl-en": 0}}
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-ps-en-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-ps-en-v0-greedy_until
new file mode 100644
index 0000000000000000000000000000000000000000..7776c5952383a6254943869dad8fddb50e50e987
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-ps-en-v0-greedy_until
@@ -0,0 +1 @@
+c3976465e3709b4bc371175cc1494c69fe096ea4ba7d114da779d2baa0a47466
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-ps-en-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-ps-en-v0-res.json
new file mode 100644
index 0000000000000000000000000000000000000000..00c9c742e4d34830628a52aa34cfd7faaa86516c
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-ps-en-v0-res.json
@@ -0,0 +1 @@
+{"results": {"wmt20-ps-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.015192865365105723, "chrf_stderr": 0.00011334541381539086, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-ps-en": 0}}
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-ru-en-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-ru-en-v0-greedy_until
new file mode 100644
index 0000000000000000000000000000000000000000..27c60fb72194325e51647ce0fe137710df6dff86
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-ru-en-v0-greedy_until
@@ -0,0 +1 @@
+1477ab6542c26bd0222cc1aded174f33bf8d04d1cf6a1c0959aeca4ff3779adc
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-ru-en-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-ru-en-v0-res.json
new file mode 100644
index 0000000000000000000000000000000000000000..b6d0c71ad7c140790040022c51970dea075ab0b1
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-ru-en-v0-res.json
@@ -0,0 +1 @@
+{"results": {"wmt20-ru-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.013344639906399232, "chrf_stderr": 7.583552652374546e-05, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-ru-en": 0}}
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-ta-en-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-ta-en-v0-greedy_until
new file mode 100644
index 0000000000000000000000000000000000000000..f0f65972451ff666399f7b2c81194c4b892ac783
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-ta-en-v0-greedy_until
@@ -0,0 +1 @@
+111ea3efdc08f1cf536631b9426c3a20e482c575d009d2a8c71f59c027578eec
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-ta-en-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-ta-en-v0-res.json
new file mode 100644
index 0000000000000000000000000000000000000000..a2ad506bf94188b54a0f3b7ee6f5d787d34e68ff
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-ta-en-v0-res.json
@@ -0,0 +1 @@
+{"results": {"wmt20-ta-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.013841110664859798, "chrf_stderr": 0.00018476696850880766, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-ta-en": 0}}
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-zh-en-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-zh-en-v0-greedy_until
new file mode 100644
index 0000000000000000000000000000000000000000..41a1e91515b30a2acdc6363c36af1cdf43f477a9
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-zh-en-v0-greedy_until
@@ -0,0 +1 @@
+07dbadfd6f2b2b9462ab6187dbfaabae6e5192ab89a8e4ede9237834b9364dd1
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wmt20-zh-en-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-zh-en-v0-res.json
new file mode 100644
index 0000000000000000000000000000000000000000..11b8df7f8739d9e4a459636640af6ebb2b7b868a
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wmt20-zh-en-v0-res.json
@@ -0,0 +1 @@
+{"results": {"wmt20-zh-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.008438201290981157, "chrf_stderr": 0.0001109053964076822, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-zh-en": 0}}
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wnli-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/wnli-v0-loglikelihood
new file mode 100644
index 0000000000000000000000000000000000000000..0c5c0b8ceb64a158bd57294d432b2186f3a0fdf9
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wnli-v0-loglikelihood
@@ -0,0 +1 @@
+2ffd304d6096416eb29607e2e7642b1d6043163624967bcf4c4fc00fddc6c721
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wnli-v0-res.json b/lm-evaluation-harness/tests/testdata/wnli-v0-res.json
new file mode 100644
index 0000000000000000000000000000000000000000..8841cb74d16977645c1c7399d8b58de094bafef1
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wnli-v0-res.json
@@ -0,0 +1 @@
+{"results": {"wnli": {"acc": 0.3380281690140845, "acc_stderr": 0.05653887739133514}}, "versions": {"wnli": 0}}
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wnli-v1-loglikelihood b/lm-evaluation-harness/tests/testdata/wnli-v1-loglikelihood
new file mode 100644
index 0000000000000000000000000000000000000000..cbf4ad3777eebbcee7c1ccf1c4a4cac64829ad2b
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wnli-v1-loglikelihood
@@ -0,0 +1 @@
+8a0f81661d2ab2334bbc8031fac31c0c8882f1d9271dd51599d21dfdbb726dea
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wnli-v1-res.json b/lm-evaluation-harness/tests/testdata/wnli-v1-res.json
new file mode 100644
index 0000000000000000000000000000000000000000..d12348e0aeb8d7feec272059e08eb30cbb1d918d
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wnli-v1-res.json
@@ -0,0 +1 @@
+{"results": {"wnli": {"acc": 0.5633802816901409, "acc_stderr": 0.0592793555841297}}, "versions": {"wnli": 1}}
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wsc-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/wsc-v0-loglikelihood
new file mode 100644
index 0000000000000000000000000000000000000000..d0d2963fe90b29dbbf2527e9a3b559cf9b9c23c7
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wsc-v0-loglikelihood
@@ -0,0 +1 @@
+45865468eff5ca31e6a050947a6b3310d9d5ed19d0f2e578a32ecaf1c768600f
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wsc-v0-res.json b/lm-evaluation-harness/tests/testdata/wsc-v0-res.json
new file mode 100644
index 0000000000000000000000000000000000000000..84be59624161779e494896d2618dbcf0f1f4b4b0
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wsc-v0-res.json
@@ -0,0 +1 @@
+{"results": {"wsc": {"acc": 0.5480769230769231, "acc_stderr": 0.049038186969314335}}, "versions": {"wsc": 0}}
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wsc273-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/wsc273-v0-loglikelihood
new file mode 100644
index 0000000000000000000000000000000000000000..9d592917bd96783c290237446891eef56083a693
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wsc273-v0-loglikelihood
@@ -0,0 +1 @@
+26450d414c4581feb51a09882080e7a9b95882e7eab47b1751a4a6024b5a60ee
\ No newline at end of file
diff --git a/lm-evaluation-harness/tests/testdata/wsc273-v0-res.json b/lm-evaluation-harness/tests/testdata/wsc273-v0-res.json
new file mode 100644
index 0000000000000000000000000000000000000000..8f023b422a7003d2984e35e58045d8866954a4c4
--- /dev/null
+++ b/lm-evaluation-harness/tests/testdata/wsc273-v0-res.json
@@ -0,0 +1 @@
+{"results": {"wsc273": {"acc": 0.5164835164835165, "acc_stderr": 0.0303004740355766}}, "versions": {"wsc273": 0}}
\ No newline at end of file
diff --git a/lmms-eval/examples/models/instructblip.sh b/lmms-eval/examples/models/instructblip.sh
new file mode 100644
index 0000000000000000000000000000000000000000..9ebc754604e013029f7cbe9f55916cfa0d0e5fc1
--- /dev/null
+++ b/lmms-eval/examples/models/instructblip.sh
@@ -0,0 +1,19 @@
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+python3 -m pip install transformers --upgrade;
+
+CKPT_PATH=$1
+TASK=$2
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+ --model instructblip \
+ --model_args pretrained=$CKPT_PATH \
+ --tasks $TASK \
+ --batch_size 1 \
+ --log_samples \
+ --log_samples_suffix instructblip \
+ --output_path ./logs/
\ No newline at end of file
diff --git a/lmms-eval/examples/models/internvl1.5.sh b/lmms-eval/examples/models/internvl1.5.sh
new file mode 100644
index 0000000000000000000000000000000000000000..a1303775511cc4b3a5fa4bae4670245f0fce9fa2
--- /dev/null
+++ b/lmms-eval/examples/models/internvl1.5.sh
@@ -0,0 +1,24 @@
+# First you need to fork [`InternVL`](https://github.com/OpenGVLab/InternVL)
+
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+cd /path/to/InternVL/internvl_chat
+python3 -m pip install -e .;
+
+python3 -m pip install flash-attn==2.3.6 --no-build-isolation;
+
+
+TASK=$1
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+ --model internvl \
+ --model_args pretrained="OpenGVLab/InternVL-Chat-V1-5"\
+ --tasks $TASK \
+ --batch_size 1 \
+ --log_samples \
+ --log_samples_suffix $TASK_SUFFIX \
+ --output_path ./logs/
\ No newline at end of file
diff --git a/lmms-eval/examples/models/llava_onevision.sh b/lmms-eval/examples/models/llava_onevision.sh
new file mode 100644
index 0000000000000000000000000000000000000000..066048ae6de3bd72ac5f3a8ec22fcec8b388e55e
--- /dev/null
+++ b/lmms-eval/examples/models/llava_onevision.sh
@@ -0,0 +1,10 @@
+export HF_HOME="~/.cache/huggingface"
+
+# pip install git+https://github.com/LLaVA-VL/LLaVA-NeXT.git
+# pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+
+accelerate launch --num_processes=8 --main_process_port 12399 -m lmms_eval \
+ --model=llava_onevision \
+ --model_args=pretrained=lmms-lab/llava-onevision-qwen2-7b-ov,conv_template=qwen_1_5,device_map=auto,model_name=llava_qwen \
+ --tasks=ai2d,chartqa,docvqa_val,mmmu_pro \
+ --batch_size=1
\ No newline at end of file
diff --git a/lmms-eval/examples/models/llava_video.sh b/lmms-eval/examples/models/llava_video.sh
new file mode 100644
index 0000000000000000000000000000000000000000..d0ba894eea9dc204bbe18e1e46a14edbae16284e
--- /dev/null
+++ b/lmms-eval/examples/models/llava_video.sh
@@ -0,0 +1,13 @@
+export HF_HOME="~/.cache/huggingface"
+
+# pip install git+https://github.com/LLaVA-VL/LLaVA-NeXT.git
+# pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+ --model llava_vid \
+ --model_args pretrained=lmms-lab/LLaVA-NeXT-Video-32B-Qwen,conv_template=qwen_1_5,video_decode_backend=decord,max_frames_num=32,mm_spatial_pool_mode=average,mm_newline_position=grid,mm_resampler_location=after \
+ --tasks videomme \
+ --batch_size 1 \
+ --log_samples \
+ --log_samples_suffix llava_vid_32B \
+ --output_path ./logs/
\ No newline at end of file
diff --git a/lmms-eval/examples/models/mplug_owl.sh b/lmms-eval/examples/models/mplug_owl.sh
new file mode 100644
index 0000000000000000000000000000000000000000..92c444da7b766d69b83fdbc36c1890101c141376
--- /dev/null
+++ b/lmms-eval/examples/models/mplug_owl.sh
@@ -0,0 +1,18 @@
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+# It has to use an old transformers version to run
+python3 -m pip install av sentencepiece protobuf==3.20 transformers==4.28.1 einops;
+
+TASK=$1
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+ --model mplug_owl_video \
+ --tasks $TASK \
+ --batch_size 1 \
+ --log_samples \
+ --log_samples_suffix $TASK_SUFFIX \
+ --output_path ./logs/
diff --git a/lmms-eval/examples/models/openai_compatible.sh b/lmms-eval/examples/models/openai_compatible.sh
new file mode 100644
index 0000000000000000000000000000000000000000..0e9dd46e4c8751a77690a7c28e82ab58561d516b
--- /dev/null
+++ b/lmms-eval/examples/models/openai_compatible.sh
@@ -0,0 +1,11 @@
+export HF_HOME="~/.cache/huggingface"
+export AZURE_OPENAI_API_KEY=""
+export AZURE_OPENAI_API_BASE=""
+export AZURE_OPENAI_API_VERSION="2023-07-01-preview"
+
+# pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+python3 -m lmms_eval \
+ --model openai_compatible \
+ --model_args model_version=gpt-4o-2024-11-20,azure_openai=True \
+ --tasks mme,mmmu_val \
+ --batch_size 1
\ No newline at end of file
diff --git a/lmms-eval/examples/models/plm.sh b/lmms-eval/examples/models/plm.sh
new file mode 100644
index 0000000000000000000000000000000000000000..6fbd4648f6341a7fff5208d3c6471d36ac752a27
--- /dev/null
+++ b/lmms-eval/examples/models/plm.sh
@@ -0,0 +1,44 @@
+# # STEP # 1: Install lmms-eval
+# pip install lmms-eval
+
+# # STEP # 2: Install perception_models (Details at https://github.com/facebookresearch/perception_models)
+# git clone https://github.com/facebookresearch/perception_models.git
+# cd perception_models
+
+# conda create --name perception_models python=3.12
+# conda activate perception_models
+
+# # Install PyTorch
+# pip install torch==2.5.1 torchvision==0.20.1 torchaudio==2.5.1 xformers --index-url https://download.pytorch.org/whl/cu124
+
+# # We use torchcodec for decoding videos into PyTorch tensors
+# conda install ffmpeg -c conda-forge
+# pip install torchcodec==0.1 --index-url=https://download.pytorch.org/whl/cu124
+
+# pip install -e .
+
+
+# Use facebook/Perception-LM-1B for 1B parameters model and facebook/Perception-LM-8B for 8B parameters model.
+CHECKPOINTS_PATH=facebook/Perception-LM-3B
+
+# Define the tasks you want to evaluate PLM on. We support all the tasks present in lmms-eval, however have tested the following tasks with our models.
+ALL_TASKS=(
+ "docvqa" "chartqa" "textvqa" "infovqa" "ai2d_no_mask" "ok_vqa" "vizwiz_vqa" "mme"
+ "realworldqa" "pope" "mmmu" "ocrbench" "coco_karpathy_val" "nocaps" "vqav2_val"
+ "mvbench" "videomme" "vatex_test" "egoschema" "egoschema_subset" "mlvu_dev"
+ "tempcompass_multi_choice" "perceptiontest_val_mc" "perceptiontest_test_mc"
+)
+
+# We select one image and one video task as an example.
+SELECTED_TASK="textvqa,videomme"
+
+# After specifying the task/tasks to evaluate, run the following command to start the evaluation.
+accelerate launch --num_processes=8 \
+-m lmms_eval \
+--model plm \
+--model_args pretrained=$CHECKPOINTS_PATH,max_tokens=11264 \
+--tasks $SELECTED_TASK \
+--batch_size 1 \
+--log_samples \
+--log_samples_suffix plm \
+--output_path plm_reproduce
diff --git a/lmms-eval/examples/models/qwen25vl.sh b/lmms-eval/examples/models/qwen25vl.sh
new file mode 100644
index 0000000000000000000000000000000000000000..e16bedf2f34c32d11bd8039d6b0b9c5e755ee449
--- /dev/null
+++ b/lmms-eval/examples/models/qwen25vl.sh
@@ -0,0 +1,18 @@
+# Run and exactly reproduce qwen2vl results!
+# mme as an example
+export HF_HOME="~/.cache/huggingface"
+# pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+# pip3 install qwen_vl_utils
+# use `interleave_visuals=True` to control the visual token position, currently only for mmmu_val and mmmu_pro (and potentially for other interleaved image-text tasks), please do not use it unless you are sure about the operation details.
+
+# accelerate launch --num_processes=8 --main_process_port=12346 -m lmms_eval \
+# --model qwen2_vl \
+# --model_args=pretrained=Qwen/Qwen2-VL-7B-Instruct,max_pixels=12845056,attn_implementation=flash_attention_2,interleave_visuals=True \
+# --tasks mmmu_pro \
+# --batch_size 1
+
+accelerate launch --num_processes=8 --main_process_port=12346 -m lmms_eval \
+ --model qwen2_5_vl \
+ --model_args=pretrained=Qwen/Qwen2.5-VL-7B-Instruct,max_pixels=12845056,attn_implementation=flash_attention_2,interleave_visuals=False \
+ --tasks mme \
+ --batch_size 1
\ No newline at end of file
diff --git a/lmms-eval/examples/models/qwen2vl.sh b/lmms-eval/examples/models/qwen2vl.sh
new file mode 100644
index 0000000000000000000000000000000000000000..eb82a2209eddd23433c5424edaabca98335e37e6
--- /dev/null
+++ b/lmms-eval/examples/models/qwen2vl.sh
@@ -0,0 +1,8 @@
+# Run and exactly reproduce qwen2vl results!
+# mme as an example
+pip3 install qwen_vl_utils
+accelerate launch --num_processes=8 --main_process_port=12345 -m lmms_eval \
+ --model qwen2_vl \
+ --model_args=pretrained=Qwen/Qwen2-VL-7B-Instruct,max_pixels=2359296 \
+ --tasks mme,gsm8k_cot_self_consistency,mmmu_val_reasoning,mmmu_val_pass64 \
+ --batch_size 1 --log_samples --log_samples_suffix reproduce --output_path ./logs/
\ No newline at end of file
diff --git a/lmms-eval/examples/models/sglang.sh b/lmms-eval/examples/models/sglang.sh
new file mode 100644
index 0000000000000000000000000000000000000000..94504a9ec9e55fc44d0cf76ee2f3d452e0ef3609
--- /dev/null
+++ b/lmms-eval/examples/models/sglang.sh
@@ -0,0 +1,12 @@
+export HF_HOME="~/.cache/huggingface"
+# pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+
+python3 -m lmms_eval \
+ --model=llava_sglang \
+ --model_args=pretrained=lmms-lab/llava-next-72b,tokenizer=lmms-lab/llavanext-qwen-tokenizer,conv_template=chatml-llava,tp_size=8,parallel=8 \
+ --tasks=mme \
+ --batch_size=1 \
+ --log_samples \
+ --log_samples_suffix=llava_qwen \
+ --output_path=./logs/ \
+ --verbosity=INFO
\ No newline at end of file
diff --git a/lmms-eval/examples/models/slime.sh b/lmms-eval/examples/models/slime.sh
new file mode 100644
index 0000000000000000000000000000000000000000..eaef444bff85211aece80d96a52e3bd7fc5df415
--- /dev/null
+++ b/lmms-eval/examples/models/slime.sh
@@ -0,0 +1,23 @@
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+git clone https://github.com/yfzhang114/SliME.git
+
+cd SliME
+pip install --upgrade pip # enable PEP 660 support
+pip install -e .
+cd ..
+
+TASK=$1
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+ --model slime \
+ --tasks $TASK \
+ --model_args pretrained="yifanzhang114/SliME-Llama3-8B,conv_template=llama3,model_name=slime" \
+ --batch_size 1 \
+ --log_samples \
+ --log_samples_suffix $TASK_SUFFIX \
+ --output_path ./logs/
\ No newline at end of file
diff --git a/lmms-eval/examples/models/tensor_parallel.sh b/lmms-eval/examples/models/tensor_parallel.sh
new file mode 100644
index 0000000000000000000000000000000000000000..74949fa6c33955cb312fec123dd755008bcdf14a
--- /dev/null
+++ b/lmms-eval/examples/models/tensor_parallel.sh
@@ -0,0 +1,12 @@
+export HF_HOME="~/.cache/huggingface"
+# pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+
+python3 -m lmms_eval \
+ --model=llava \
+ --model_args=pretrained=lmms-lab/llava-next-72b,conv_template=qwen_1_5,device_map=auto,model_name=llava_qwen \
+ --tasks=pope,vizwiz_vqa_val,scienceqa_img \
+ --batch_size=1 \
+ --log_samples \
+ --log_samples_suffix=llava_qwen \
+ --output_path="./logs/" \
+ --wandb_args=project=lmms-eval,job_type=eval,entity=llava-vl
\ No newline at end of file
diff --git a/lmms-eval/examples/models/tinyllava.sh b/lmms-eval/examples/models/tinyllava.sh
new file mode 100644
index 0000000000000000000000000000000000000000..902ff0d4e48a4ab51a9d73b47c14a68890b9e2e1
--- /dev/null
+++ b/lmms-eval/examples/models/tinyllava.sh
@@ -0,0 +1,25 @@
+# install lmms_eval without building dependencies
+cd lmms_eval;
+pip install --no-deps -U -e .
+
+# install TinyLLaVA without building dependencies
+cd ..
+git clone https://github.com/TinyLLaVA/TinyLLaVA_Factory
+cd TinyLLaVA_Factory
+pip install --no-deps -U -e .
+
+# install all the requirements that require for reproduce llava results
+pip install torch==2.0.1 torchvision==0.15.2 --index-url https://download.pytorch.org/whl/cu118
+pip install -r tinyllava_repr_requirements.txt
+
+# Run and reproduce tinyllava best results!
+accelerate launch \
+ --num_processes=1 \
+ -m lmms_eval \
+ --model tinyllava \
+ --model_args pretrained=tinyllava/TinyLLaVA-Phi-2-SigLIP-3.1B,conv_mode=phi \
+ --tasks vqav2,gqa,scienceqa_img,textvqa,mmvet,pope,mme,mmmu_val \
+ --batch_size 1 \
+ --log_samples \
+ --log_samples_suffix tinyllava-phi2-siglip-3.1b \
+ --output_path ./logs/
\ No newline at end of file
diff --git a/lmms-eval/examples/models/video_chatgpt.sh b/lmms-eval/examples/models/video_chatgpt.sh
new file mode 100644
index 0000000000000000000000000000000000000000..beb30db628561c7fcc7f2c705d587de3001a39a4
--- /dev/null
+++ b/lmms-eval/examples/models/video_chatgpt.sh
@@ -0,0 +1,17 @@
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+python3 -m pip install sentencepiece av;
+
+TASK=$1
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+ --model video_chatgpt \
+ --tasks $TASK \
+ --batch_size 1 \
+ --log_samples \
+ --log_samples_suffix $TASK_SUFFIX \
+ --output_path ./logs/
\ No newline at end of file
diff --git a/lmms-eval/examples/models/video_llava.sh b/lmms-eval/examples/models/video_llava.sh
new file mode 100644
index 0000000000000000000000000000000000000000..f4beaad28e2b1345bda388a1e0428cd2ecea8c3f
--- /dev/null
+++ b/lmms-eval/examples/models/video_llava.sh
@@ -0,0 +1,19 @@
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+python3 -m pip install transformers --upgrade;
+python3 -m pip install av sentencepiece;
+
+
+TASK=$1
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+ --model video_llava \
+ --tasks $TASK \
+ --batch_size 1 \
+ --log_samples \
+ --log_samples_suffix $TASK_SUFFIX \
+ --output_path ./logs/
\ No newline at end of file
diff --git a/lmms-eval/examples/models/vllm_qwen2vl.sh b/lmms-eval/examples/models/vllm_qwen2vl.sh
new file mode 100644
index 0000000000000000000000000000000000000000..e192f0a362888328738695b7cbb9eae6b6477eda
--- /dev/null
+++ b/lmms-eval/examples/models/vllm_qwen2vl.sh
@@ -0,0 +1,19 @@
+
+# pip3 install vllm
+# pip3 install qwen_vl_utils
+
+# cd ~/prod/lmms-eval-public
+# pip3 install -e .
+export NCCL_BLOCKING_WAIT=1
+export NCCL_TIMEOUT=18000000
+export NCCL_DEBUG=DEBUG
+
+python3 -m lmms_eval \
+ --model vllm \
+ --model_args model=Qwen/Qwen2-VL-7B-Instruct,tensor_parallel_size=4 \
+ --tasks mme,gsm8k_cot_self_consistency,mmmu_val \
+ --batch_size 64 \
+ --log_samples \
+ --log_samples_suffix vllm \
+ --output_path ./logs \
+ --limit=64
\ No newline at end of file
diff --git a/lmms-eval/examples/models/xai_grok.sh b/lmms-eval/examples/models/xai_grok.sh
new file mode 100644
index 0000000000000000000000000000000000000000..b6af3026069b09b70b4af47712e1ea12be4ac78c
--- /dev/null
+++ b/lmms-eval/examples/models/xai_grok.sh
@@ -0,0 +1,10 @@
+export HF_HOME="~/.cache/huggingface"
+export OPENAI_API_KEY="xai-xxxxxxxxxx"
+export OPENAI_API_BASE="https://api.x.ai/v1"
+
+
+python3 -m lmms_eval \
+ --model openai_compatible \
+ --model_args model_version=grok-2-vision-1212 \
+ --tasks ai2d,chartqa,docvqa_val,mathvista_testmini,mmmu_pro \
+ --batch_size 1
\ No newline at end of file
diff --git a/lmms-eval/examples/models/xcomposer.sh b/lmms-eval/examples/models/xcomposer.sh
new file mode 100644
index 0000000000000000000000000000000000000000..390ae84aebcb77e998d27e57a29a915ceadcaf12
--- /dev/null
+++ b/lmms-eval/examples/models/xcomposer.sh
@@ -0,0 +1,32 @@
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+
+python3 -m pip install flash-attn --no-build-isolation;
+python3 -m pip install torchvision einops timm sentencepiece;
+
+TASK=$1
+MODALITY=$2
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+# For Xcomposer2d5
+accelerate launch --num_processes 8 --main_process_port 10000 -m lmms_eval \
+ --model xcomposer2d5 \
+ --model_args pretrained="internlm/internlm-xcomposer2d5-7b",device="cuda",modality=$MODALITY\
+ --tasks $TASK \
+ --batch_size 1 \
+ --log_samples \
+ --log_samples_suffix $TASK_SUFFIX \
+ --output_path ./logs/
+
+# For Xcomposer-4kHD
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+ --model xcomposer2_4khd \
+ --model_args pretrained="internlm/internlm-xcomposer2-4khd-7b" \
+ --tasks $TASK \
+ --batch_size 1 \
+ --log_samples \
+ --log_samples_suffix $TASK_SUFFIX \
+ --output_path ./logs/
\ No newline at end of file
diff --git a/lmms-eval/lmms_eval/__init__.py b/lmms-eval/lmms_eval/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lmms-eval/lmms_eval/__main__.py b/lmms-eval/lmms_eval/__main__.py
new file mode 100644
index 0000000000000000000000000000000000000000..c10839f36423ec2e2460437d6cf416d089ca2df2
--- /dev/null
+++ b/lmms-eval/lmms_eval/__main__.py
@@ -0,0 +1,539 @@
+import argparse
+import datetime
+import importlib
+import json
+import os
+import sys
+import traceback
+import warnings
+from functools import partial
+
+import numpy as np
+import torch
+import yaml
+
+warnings.simplefilter("ignore", category=DeprecationWarning)
+
+import hashlib
+from pathlib import Path
+from typing import Union
+
+from accelerate import Accelerator
+from accelerate.utils import InitProcessGroupKwargs
+from loguru import logger as eval_logger
+
+from lmms_eval import evaluator, utils
+from lmms_eval.api.registry import ALL_TASKS
+from lmms_eval.evaluator import request_caching_arg_to_dict
+from lmms_eval.loggers import EvaluationTracker, WandbLogger
+from lmms_eval.tasks import TaskManager
+from lmms_eval.utils import (
+ handle_non_serializable,
+ make_table,
+ simple_parse_args_string,
+)
+
+
+def _int_or_none_list_arg_type(min_len: int, max_len: int, defaults: str, value: str, split_char: str = ","):
+ def parse_value(item):
+ item = item.strip().lower()
+ if item == "none":
+ return None
+ try:
+ return int(item)
+ except ValueError:
+ raise argparse.ArgumentTypeError(f"{item} is not an integer or None")
+
+ items = [parse_value(v) for v in value.split(split_char)]
+ num_items = len(items)
+
+ if num_items == 1:
+ # Makes downstream handling the same for single and multiple values
+ items = items * max_len
+ elif num_items < min_len or num_items > max_len:
+ raise argparse.ArgumentTypeError(f"Argument requires {max_len} integers or None, separated by '{split_char}'")
+ elif num_items != max_len:
+ logging.warning(f"Argument requires {max_len} integers or None, separated by '{split_char}'. " "Missing values will be filled with defaults.")
+ default_items = [parse_value(v) for v in defaults.split(split_char)]
+ items.extend(default_items[num_items:]) # extend items list with missing defaults
+
+ return items
+
+
+def check_argument_types(parser: argparse.ArgumentParser):
+ """
+ Check to make sure all CLI args are typed, raises error if not
+ """
+ for action in parser._actions:
+ if action.dest != "help" and not action.const:
+ if action.type is None:
+ raise ValueError(f"Argument '{action.dest}' doesn't have a type specified.")
+ else:
+ continue
+
+
+def _handle_non_serializable(o):
+ if isinstance(o, np.int64) or isinstance(o, np.int32):
+ return int(o)
+ elif isinstance(o, set):
+ return list(o)
+ else:
+ return str(o)
+
+
+def parse_eval_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
+ parser.add_argument("--config", default="", help="Path to a yaml file specifying all eval arguments, will ignore cli arguments if specified")
+ parser.add_argument("--model", default="hf", help="Name of model e.g. `hf`")
+ parser.add_argument(
+ "--tasks",
+ default=None,
+ help="To get full list of tasks, use the command lmms-eval --tasks list",
+ )
+ parser.add_argument(
+ "--model_args",
+ default="",
+ help="String arguments for model, e.g. `pretrained=EleutherAI/pythia-160m,dtype=float32`",
+ )
+ parser.add_argument(
+ "--launcher_args",
+ default=None,
+ help="String arguments for launcher for local llm as judge, e.g. `tp=8`, if None then no launcher will be used.",
+ )
+ parser.add_argument(
+ "--num_fewshot",
+ type=int,
+ default=None,
+ help="Number of examples in few-shot context",
+ )
+ parser.add_argument(
+ "--batch_size",
+ "-b",
+ type=str,
+ default=1,
+ metavar="auto|auto:N|N",
+ help="Acceptable values are 'auto', 'auto:N' or N, where N is an integer. Default 1.",
+ )
+ parser.add_argument(
+ "--max_batch_size",
+ type=int,
+ default=None,
+ metavar="N",
+ help="Maximal batch size to try with --batch_size auto.",
+ )
+ parser.add_argument(
+ "--device",
+ type=str,
+ default=None,
+ help="Device to use (e.g. cuda, cuda:0, cpu)",
+ )
+ parser.add_argument(
+ "--output_path",
+ default=None,
+ type=str,
+ metavar="= [dir/file.jsonl] [DIR]",
+ help="The path to the output file where the result metrics will be saved. If the path is a directory and log_samples is true, the results will be saved in the directory. Else the parent directory will be used.",
+ )
+ parser.add_argument(
+ "--limit",
+ type=float,
+ default=None,
+ help="Limit the number of examples per task. " "If <1, limit is a percentage of the total number of examples.",
+ )
+ parser.add_argument(
+ "--use_cache",
+ "-c",
+ type=str,
+ default=None,
+ metavar="DIR",
+ help="A path to a sqlite db file for caching model responses. `None` if not caching.",
+ )
+ parser.add_argument(
+ "--cache_requests",
+ type=str,
+ default=None,
+ choices=["true", "refresh", "delete"],
+ help="Speed up evaluation by caching the building of dataset requests. `None` if not caching.",
+ )
+ parser.add_argument(
+ "--check_integrity",
+ action="store_true",
+ help="Whether to run the relevant part of the test suite for the tasks",
+ )
+ parser.add_argument(
+ "--write_out",
+ "-w",
+ action="store_true",
+ default=False,
+ help="Prints the prompt for the first few documents.",
+ )
+ parser.add_argument(
+ "--log_samples",
+ action="store_true",
+ default=False,
+ help="If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis",
+ )
+ parser.add_argument(
+ "--wandb_log_samples",
+ action="store_true",
+ default=False,
+ help="If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis to Weights and Biases",
+ )
+ parser.add_argument(
+ "--log_samples_suffix",
+ type=str,
+ default="model_outputs",
+ help="Specify a suffix for the log_samples file name.",
+ )
+ parser.add_argument(
+ "--system_instruction",
+ type=str,
+ default=None,
+ help="System instruction to be used in the prompt",
+ )
+ parser.add_argument(
+ "--apply_chat_template",
+ action="store_true",
+ default=False,
+ help="If True, applies the chat template to the prompt",
+ )
+ parser.add_argument(
+ "--fewshot_as_multiturn",
+ action="store_true",
+ default=False,
+ help="If True, uses the fewshot as a multi-turn conversation",
+ )
+ parser.add_argument(
+ "--show_config",
+ action="store_true",
+ default=False,
+ help="If True, shows the the full config of all tasks at the end of the evaluation.",
+ )
+ parser.add_argument(
+ "--include_path",
+ type=str,
+ default=None,
+ help="Additional path to include if there are external tasks to include.",
+ )
+ parser.add_argument(
+ "--gen_kwargs",
+ default="",
+ help=("String arguments for model generation on greedy_until tasks," " e.g. `temperature=0,top_k=0,top_p=0`"),
+ )
+ parser.add_argument(
+ "--verbosity",
+ type=str,
+ default="INFO",
+ help="Log error when tasks are not registered.",
+ )
+ parser.add_argument(
+ "--wandb_args",
+ default="",
+ help="Comma separated string arguments passed to wandb.init, e.g. `project=lmms-eval,job_type=eval",
+ )
+ parser.add_argument(
+ "--timezone",
+ default="Asia/Singapore",
+ help="Timezone for datetime string, e.g. Asia/Singapore, America/New_York, America/Los_Angeles. You can check the full list via `import pytz; print(pytz.common_timezones)`",
+ )
+ parser.add_argument(
+ "--hf_hub_log_args",
+ type=str,
+ default="",
+ help="Comma separated string arguments passed to Hugging Face Hub's log function, e.g. `hub_results_org=EleutherAI,hub_repo_name=lm-eval-results`",
+ )
+ parser.add_argument(
+ "--predict_only",
+ "-x",
+ action="store_true",
+ default=False,
+ help="Use with --log_samples. Only model outputs will be saved and metrics will not be evaluated.",
+ )
+ default_seed_string = "0,1234,1234,1234"
+ parser.add_argument(
+ "--seed",
+ type=partial(_int_or_none_list_arg_type, 3, 4, default_seed_string),
+ default=default_seed_string, # for backward compatibility
+ help=(
+ "Set seed for python's random, numpy, torch, and fewshot sampling.\n"
+ "Accepts a comma-separated list of 4 values for python's random, numpy, torch, and fewshot sampling seeds, "
+ "respectively, or a single integer to set the same seed for all four.\n"
+ f"The values are either an integer or 'None' to not set the seed. Default is `{default_seed_string}` "
+ "(for backward compatibility).\n"
+ "E.g. `--seed 0,None,8,52` sets `random.seed(0)`, `torch.manual_seed(8)`, and fewshot sampling seed to 52. "
+ "Here numpy's seed is not set since the second value is `None`.\n"
+ "E.g, `--seed 42` sets all four seeds to 42."
+ ),
+ )
+ parser.add_argument(
+ "--trust_remote_code",
+ action="store_true",
+ help="Sets trust_remote_code to True to execute code to create HF Datasets from the Hub",
+ )
+ parser.add_argument("--process_with_media", action="store_true", help="Whether you will process you dataset with audio, image. By default set to False" "In case some benchmarks need to be processed with media, set this flag to True.")
+ parser.add_argument("--force_simple", action="store_true", help="Force the evaluation to use the simple mode of the models")
+ args = parser.parse_args()
+ return args
+
+
+def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
+ default_args = parse_eval_args()
+
+ if args is None and len(sys.argv) == 1:
+ print("┌───────────────────────────────────────────────────────────────────────────────┐")
+ print("│ Please provide arguments to evaluate the model. e.g. │")
+ print("│ `lmms-eval --model llava --model_path liuhaotian/llava-v1.6-7b --tasks okvqa` │")
+ print("│ Use `lmms-eval --help` for more information. │")
+ print("└───────────────────────────────────────────────────────────────────────────────┘")
+ sys.exit(1)
+
+ # If args were provided, override the defaults
+ if args:
+ for key, value in vars(args).items():
+ setattr(default_args, key, value)
+
+ args = default_args
+
+ if args.wandb_args:
+ if "name" not in args.wandb_args:
+ name = f"{args.model}_{args.model_args}_{utils.get_datetime_str(timezone=args.timezone)}"
+ name = utils.sanitize_long_string(name)
+ args.wandb_args += f",name={name}"
+ wandb_logger = WandbLogger(**simple_parse_args_string(args.wandb_args))
+
+ # reset logger
+ eval_logger.remove()
+ # Configure logger with detailed format including file path, function name, and line number
+ log_format = "{time:YYYY-MM-DD HH:mm:ss} | " "{level: <8} | " "{name}:{function}:{line} - " "{message}"
+ eval_logger.add(sys.stdout, colorize=True, level=args.verbosity, format=log_format)
+ eval_logger.info(f"Verbosity set to {args.verbosity}")
+ os.environ["VERBOSITY"] = args.verbosity
+
+ args_list = []
+ results_list = []
+ if args.config:
+ if not os.path.exists(args.config):
+ raise ValueError(f"Config file does not exist: {args.config}")
+
+ with open(args.config, "r") as file:
+ config_args = yaml.safe_load(file)
+ config_args = [config_args] if type(config_args) != list else config_args
+ # multiple configs, create args list first
+ for config in config_args:
+ args_copy = argparse.Namespace(**vars(args))
+ for key, value in config.items():
+ setattr(args_copy, key, value)
+ args_list.append(args_copy)
+ else:
+ args_list.append(args)
+
+ # initialize Accelerator only if not already in a distributed context
+ if torch.distributed.is_available() and torch.distributed.is_initialized():
+ accelerator = None
+ is_main_process = torch.distributed.get_rank() == 0
+ else:
+ kwargs_handler = InitProcessGroupKwargs(timeout=datetime.timedelta(seconds=60000))
+ accelerator = Accelerator(kwargs_handlers=[kwargs_handler])
+ if accelerator.is_main_process:
+ is_main_process = True
+ else:
+ is_main_process = False
+
+ for args in args_list:
+ try:
+ # if is_main_process and args.wandb_args: # thoughtfully we should only init wandb once, instead of multiple ranks to avoid network traffics and unwanted behaviors.
+ # wandb_logger = WandbLogger()
+
+ results, samples = cli_evaluate_single(args)
+ results_list.append(results)
+
+ if accelerator:
+ accelerator.wait_for_everyone()
+ elif torch.distributed.is_available() and torch.distributed.is_initialized():
+ torch.distributed.barrier()
+ if is_main_process and args.wandb_args:
+ try:
+ wandb_logger.post_init(results)
+ wandb_logger.log_eval_result()
+ if args.wandb_log_samples and samples is not None:
+ wandb_logger.log_eval_samples(samples)
+ except Exception as e:
+ eval_logger.info(f"Logging to Weights and Biases failed due to {e}")
+ # wandb_logger.finish()
+
+ except Exception as e:
+ if args.verbosity == "DEBUG":
+ raise e
+ else:
+ traceback.print_exc()
+ eval_logger.error(f"Error during evaluation: {e}. Please set `--verbosity=DEBUG` to get more information.")
+ results_list.append(None)
+
+ for args, results in zip(args_list, results_list):
+ # cli_evaluate will return none if the process is not the main process (rank 0)
+ if results is not None:
+ print(f"{args.model} ({args.model_args}), gen_kwargs: ({args.gen_kwargs}), limit: {args.limit}, num_fewshot: {args.num_fewshot}, " f"batch_size: {args.batch_size}")
+ print(make_table(results))
+ if "groups" in results:
+ print(make_table(results, "groups"))
+
+ if args.wandb_args:
+ wandb_logger.run.finish()
+
+
+def cli_evaluate_single(args: Union[argparse.Namespace, None] = None) -> None:
+ selected_task_list = args.tasks.split(",") if args.tasks else None
+
+ if args.include_path is not None:
+ eval_logger.info(f"Including path: {args.include_path}")
+ task_manager = TaskManager(args.verbosity, include_path=args.include_path, model_name=args.model)
+
+ # update the evaluation tracker args with the output path and the HF token
+ if args.output_path:
+ args.hf_hub_log_args += f",output_path={args.output_path}"
+ if os.environ.get("HF_TOKEN", None):
+ args.hf_hub_log_args += f",token={os.environ.get('HF_TOKEN')}"
+
+ evaluation_tracker_args = simple_parse_args_string(args.hf_hub_log_args)
+ eval_logger.info(f"Evaluation tracker args: {evaluation_tracker_args}")
+
+ evaluation_tracker = EvaluationTracker(**evaluation_tracker_args)
+
+ if args.predict_only:
+ args.log_samples = True
+ if (args.log_samples or args.predict_only) and not args.output_path:
+ raise ValueError("Specify --output_path if providing --log_samples or --predict_only")
+
+ if args.fewshot_as_multiturn and args.apply_chat_template is False:
+ raise ValueError("If fewshot_as_multiturn is set, apply_chat_template must be set to True.")
+
+ if (args.num_fewshot is None or args.num_fewshot == 0) and args.fewshot_as_multiturn:
+ raise ValueError("If fewshot_as_multiturn is set, num_fewshot must be greater than 0.")
+
+ if args.include_path is not None:
+ eval_logger.info(f"Including path: {args.include_path}")
+
+ if "push_samples_to_hub" in evaluation_tracker_args and not args.log_samples:
+ eval_logger.warning("Pushing samples to the Hub requires --log_samples to be set. Samples will not be pushed to the Hub.")
+
+ if args.limit:
+ eval_logger.warning(" --limit SHOULD ONLY BE USED FOR TESTING." "REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")
+
+ if os.environ.get("LMMS_EVAL_PLUGINS", None):
+ args.include_path = [args.include_path] if args.include_path else []
+ for plugin in os.environ["LMMS_EVAL_PLUGINS"].split(","):
+ package_tasks_location = importlib.util.find_spec(f"{plugin}.tasks").submodule_search_locations[0]
+ args.include_path.append(package_tasks_location)
+
+ if args.tasks is None:
+ eval_logger.error("Need to specify task to evaluate.")
+ sys.exit()
+ elif args.tasks == "list":
+ eval_logger.info("Available Tasks:\n - {}".format(f"\n - ".join(sorted(task_manager.all_tasks))))
+ sys.exit()
+ elif args.tasks == "list_groups":
+ eval_logger.info(task_manager.list_all_tasks(list_subtasks=False, list_tags=False))
+ sys.exit()
+ elif args.tasks == "list_tags":
+ eval_logger.info(task_manager.list_all_tasks(list_groups=False, list_subtasks=False))
+ sys.exit()
+ elif args.tasks == "list_subtasks":
+ eval_logger.info(task_manager.list_all_tasks(list_groups=False, list_tags=False))
+ sys.exit()
+ else:
+ if os.path.isdir(args.tasks):
+ import glob
+
+ task_names = []
+ yaml_path = os.path.join(args.tasks, "*.yaml")
+ for yaml_file in glob.glob(yaml_path):
+ config = utils.load_yaml_config(yaml_file)
+ task_names.append(config)
+ else:
+ task_list = args.tasks.split(",")
+ task_names = task_manager.match_tasks(task_list)
+ for task in [task for task in task_list if task not in task_names]:
+ if os.path.isfile(task):
+ config = utils.load_yaml_config(task)
+ task_names.append(config)
+ task_missing = [task for task in task_list if task not in task_names and "*" not in task] # we don't want errors if a wildcard ("*") task name was used
+
+ if task_missing:
+ missing = ", ".join(task_missing)
+ eval_logger.error(
+ f"Tasks were not found: {missing}\n" f"{utils.SPACING}Try `lmms-eval --tasks list` for list of available tasks",
+ )
+ raise ValueError(
+ f"Tasks not found: {missing}. Try `lmms-eval --tasks {{list_groups,list_subtasks,list_tags,list}}` to list out all available names for task groupings; only (sub)tasks; tags; or all of the above, or pass '--verbosity DEBUG' to troubleshoot task registration issues."
+ )
+
+ eval_logger.info(f"Selected Tasks: {task_names}")
+ request_caching_args = request_caching_arg_to_dict(cache_requests=args.cache_requests)
+ datetime_str = utils.get_datetime_str(timezone=args.timezone)
+
+ results = evaluator.simple_evaluate(
+ model=args.model,
+ model_args=args.model_args,
+ tasks=task_names,
+ num_fewshot=args.num_fewshot,
+ batch_size=args.batch_size,
+ max_batch_size=args.max_batch_size,
+ device=args.device,
+ use_cache=args.use_cache,
+ limit=args.limit,
+ check_integrity=args.check_integrity,
+ write_out=args.write_out,
+ log_samples=args.log_samples,
+ evaluation_tracker=evaluation_tracker,
+ system_instruction=args.system_instruction,
+ apply_chat_template=args.apply_chat_template,
+ fewshot_as_multiturn=args.fewshot_as_multiturn,
+ gen_kwargs=args.gen_kwargs,
+ task_manager=task_manager,
+ verbosity=args.verbosity,
+ predict_only=args.predict_only,
+ random_seed=args.seed[0],
+ numpy_random_seed=args.seed[1],
+ torch_random_seed=args.seed[2],
+ fewshot_random_seed=args.seed[3],
+ cli_args=args,
+ datetime_str=datetime_str,
+ distributed_executor_backend="torchrun" if (torch.distributed.is_available() and torch.distributed.is_initialized()) else "accelerate",
+ force_simple=args.force_simple,
+ launcher_args=args.launcher_args,
+ **request_caching_args,
+ )
+
+ if results is not None:
+ if args.log_samples:
+ samples = results.pop("samples")
+ else:
+ samples = None
+ dumped = json.dumps(results, indent=4, default=_handle_non_serializable)
+ if args.show_config:
+ print(dumped)
+
+ batch_sizes = ",".join(map(str, results["config"]["batch_sizes"]))
+
+ evaluation_tracker.save_results_aggregated(results=results, samples=samples if args.log_samples else None, datetime_str=datetime_str)
+
+ if args.log_samples:
+ for task_name, config in results["configs"].items():
+ evaluation_tracker.save_results_samples(task_name=task_name, samples=samples[task_name])
+
+ if evaluation_tracker.push_results_to_hub or evaluation_tracker.push_samples_to_hub:
+ evaluation_tracker.recreate_metadata_card()
+
+ return results, samples
+ return None, None
+
+
+def print_results(args, results):
+ print(f"{args.model} ({args.model_args}),\ngen_kwargs: ({args.gen_kwargs}),\nlimit: {args.limit},\nnum_fewshot: {args.num_fewshot},\nbatch_size: {args.batch_size}")
+ print(evaluator.make_table(results))
+ if "groups" in results:
+ print(evaluator.make_table(results, "groups"))
+
+
+if __name__ == "__main__":
+ cli_evaluate()
diff --git a/lmms-eval/lmms_eval/api/__init__.py b/lmms-eval/lmms_eval/api/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lmms-eval/lmms_eval/api/filter.py b/lmms-eval/lmms_eval/api/filter.py
new file mode 100644
index 0000000000000000000000000000000000000000..654ea6332d852bf10abc6e8426fc61711d01d6fe
--- /dev/null
+++ b/lmms-eval/lmms_eval/api/filter.py
@@ -0,0 +1,54 @@
+from dataclasses import dataclass
+from typing import List
+
+from datasets import Dataset
+
+from lmms_eval.api.instance import Instance
+
+
+class Filter:
+ """
+ Filter classes operate on a per-task level.
+ They take all model outputs (`instance.resps` for all `task.instances`)
+ across all instances of a task, and perform operations.
+ In a single run, one can configure any number of separate filters or lists of filters.
+
+ """
+
+ def __init__(self, *args, **kwargs) -> None:
+ """
+ Can define custom behavior here, if an individual instantiation of a Filter class should have state.
+ """
+
+ def apply(self, resps, docs):
+ """
+ Defines the operation to perform on a list of the `inst.resps` properties of `Instance` objects.
+ Should return the list of (filtered) response lists *in the same order as they were input*, e.g.
+ if pass in [, ] should return
+ [, ]
+ """
+ return resps
+
+
+@dataclass
+class FilterEnsemble:
+ """
+ FilterEnsemble creates a pipeline applying multiple filters.
+ Its intended usage is to stack multiple post-processing steps in order.
+ `task.apply_filters` should use a list of FilterEnsemble classes that it stores, to apply each
+ pipeline separately.
+ """
+
+ name: str
+ filters: List[Filter]
+
+ def apply(self, instances: List[Instance], docs: List[Dataset]) -> None:
+ resps = [inst.resps for inst in instances] # operate just on the model responses
+ for f in self.filters:
+ # apply filters in sequence
+ resps = f.apply(resps, docs)
+
+ # add the end results after filtering to filtered_requests of their respective source instances.
+ # has key `self.name`: each FilterEnsemble applied in a given run should use a different name.
+ for inst, resp in zip(instances, resps):
+ inst.filtered_resps[self.name] = resp
diff --git a/lmms-eval/lmms_eval/api/group.py b/lmms-eval/lmms_eval/api/group.py
new file mode 100644
index 0000000000000000000000000000000000000000..c03f805af2786ec2016520a9b9dc26483440bcf4
--- /dev/null
+++ b/lmms-eval/lmms_eval/api/group.py
@@ -0,0 +1,104 @@
+import abc
+from dataclasses import asdict, dataclass
+from inspect import getsource
+from typing import Any, Callable, List, Optional, Union
+
+
+@dataclass
+class AggMetricConfig(dict):
+ metric: Optional[str] = None
+ aggregation: Optional[str] = "mean"
+ weight_by_size: Optional[str] = False
+ # list of filter names which should be incorporated into the aggregated metric.
+ filter_list: Optional[Union[str, list]] = "none"
+
+ def __post_init__(self):
+ if self.aggregation != "mean" and not callable(self.aggregation):
+ raise ValueError(f"Currently, 'mean' is the only pre-defined aggregation across groups' subtasks. Got '{self.aggregation}'.")
+
+ if isinstance(self.filter_list, str):
+ self.filter_list = [self.filter_list]
+
+
+@dataclass
+class GroupConfig(dict):
+ group: Optional[str] = None
+ group_alias: Optional[str] = None
+ task: Optional[Union[str, list]] = None
+ aggregate_metric_list: Optional[Union[List[AggMetricConfig], AggMetricConfig, dict]] = None
+ metadata: Optional[dict] = None # by default, not used in the code. allows for users to pass arbitrary info to tasks
+
+ def __getitem__(self, item):
+ return getattr(self, item)
+
+ def __setitem__(self, item, value):
+ return setattr(self, item, value)
+
+ def __post_init__(self):
+ if self.aggregate_metric_list is not None:
+ if isinstance(self.aggregate_metric_list, dict):
+ self.aggregate_metric_list = [self.aggregate_metric_list]
+
+ self.aggregate_metric_list = [AggMetricConfig(**item) if isinstance(item, dict) else item for item in self.aggregate_metric_list]
+
+ def to_dict(self, keep_callable: bool = False) -> dict:
+ """dumps the current config as a dictionary object, as a printable format.
+ null fields will not be printed.
+ Used for dumping results alongside full task configuration
+
+ :return: dict
+ A printable dictionary version of the TaskConfig object.
+
+ # TODO: should any default value in the TaskConfig not be printed?
+ """
+ cfg_dict = asdict(self)
+ # remove values that are `None`
+ for k, v in list(cfg_dict.items()):
+ if callable(v):
+ cfg_dict[k] = self.serialize_function(v, keep_callable=keep_callable)
+ return cfg_dict
+
+ def serialize_function(self, value: Union[Callable, str], keep_callable=False) -> Union[Callable, str]:
+ """Serializes a given function or string.
+
+ If 'keep_callable' is True, the original callable is returned.
+ Otherwise, attempts to return the source code of the callable using 'getsource'.
+ """
+ if keep_callable:
+ return value
+ else:
+ try:
+ return getsource(value)
+ except (TypeError, OSError):
+ return str(value)
+
+
+class ConfigurableGroup(abc.ABC):
+ def __init__(
+ self,
+ config: Optional[dict] = None,
+ ) -> None:
+ self._config = GroupConfig(**config)
+
+ @property
+ def group(self):
+ return self._config.group
+
+ @property
+ def group_alias(self):
+ return self._config.group_alias
+
+ @property
+ def version(self):
+ return self._config.version
+
+ @property
+ def config(self):
+ return self._config.to_dict()
+
+ @property
+ def group_name(self) -> Any:
+ return self._config.group
+
+ def __repr__(self):
+ return f"ConfigurableGroup(group={self.group}," f"group_alias={self.group_alias})"
diff --git a/lmms-eval/lmms_eval/api/instance.py b/lmms-eval/lmms_eval/api/instance.py
new file mode 100644
index 0000000000000000000000000000000000000000..18cfb73990f82f31c70d4e613dd42acde44a1e25
--- /dev/null
+++ b/lmms-eval/lmms_eval/api/instance.py
@@ -0,0 +1,29 @@
+from dataclasses import dataclass, field
+from typing import Literal, Tuple
+
+
+@dataclass
+class Instance:
+ request_type: Literal["loglikelihood", "generate_until", "generate_until_multi_round"]
+ arguments: tuple
+ idx: int
+ metadata: Tuple[str, int, int] = field(default_factory=lambda: (None, None, None)) # TODO: better typehints here
+ resps: list = field(default_factory=list)
+ filtered_resps: dict = field(default_factory=dict)
+
+ # initialized after init
+ task_name: str = None
+ doc_id: str = None
+ repeats: str = None
+ doc: dict = None
+
+ def __post_init__(self) -> None:
+ # unpack metadata field
+ self.task_name, self.doc_id, self.repeats = self.metadata["task"], self.metadata["doc_id"], self.metadata["repeats"]
+
+ @property
+ def args(self):
+ """
+ Returns (string,) where `string` is the string to calculate loglikelihood over
+ """
+ return self.arguments if isinstance(self.arguments, tuple) else (self.arguments,)
diff --git a/lmms-eval/lmms_eval/api/metrics.py b/lmms-eval/lmms_eval/api/metrics.py
new file mode 100644
index 0000000000000000000000000000000000000000..e5b4214c64521a0c6be018252da433c0d1ce80fa
--- /dev/null
+++ b/lmms-eval/lmms_eval/api/metrics.py
@@ -0,0 +1,606 @@
+# the code is adapted from https://github.com/EleutherAI/lm-evaluation-harness
+import math
+import random
+import re
+import string
+from collections.abc import Iterable
+from typing import List
+
+import numpy as np
+import sacrebleu
+from loguru import logger as eval_logger
+
+from lmms_eval.api.registry import register_aggregation, register_metric
+
+
+# Register Aggregations First
+@register_aggregation("bypass")
+def bypass_agg(arr):
+ return 999
+
+
+@register_aggregation("mean")
+def mean(arr):
+ return sum(arr) / len(arr)
+
+
+@register_aggregation("median")
+def median(arr):
+ return arr[len(arr) // 2]
+
+
+# Certain metrics must be calculated across all documents in a benchmark.
+# We use them as aggregation metrics, paired with no-op passthrough metric fns.
+@register_aggregation("perplexity")
+def perplexity(items):
+ return math.exp(-mean(items))
+
+
+@register_aggregation("weighted_perplexity")
+def weighted_perplexity(items):
+ return math.exp(-weighted_mean(items))
+
+
+@register_aggregation("bits_per_byte")
+def bits_per_byte(items):
+ return -weighted_mean(items) / math.log(2)
+
+
+@register_aggregation("f1")
+def f1_score(items):
+ from sklearn.metrics import f1_score
+
+ unzipped_list = list(zip(*items))
+ golds = unzipped_list[0]
+ preds = unzipped_list[1]
+ fscore = f1_score(golds, preds)
+
+ return np.max(fscore)
+
+
+@register_aggregation("matthews_corrcoef")
+def matthews_corrcoef(items):
+ from sklearn.metrics import matthews_corrcoef
+
+ unzipped_list = list(zip(*items))
+ golds = unzipped_list[0]
+ preds = unzipped_list[1]
+ return matthews_corrcoef(golds, preds)
+
+
+@register_aggregation("bleu")
+def bleu(items):
+ """The Bilingual Evaluation Understudy Score, or BLEU for short, is a metric
+ for evaluating a generated sentence to a reference sentence. It counts matching
+ n-grams in the candidate translation to n-grams in the reference text, where
+ 1-gram or unigram would be each token and a bigram comparison would be each
+ word pair. The comparison is made regardless of word order
+ Source: https://machinelearningmastery.com/calculate-bleu-score-for-text-python/
+ Paper: https://www.aclweb.org/anthology/P02-1040/
+
+ Higher is better
+ """
+ refs = list(zip(*items))[0]
+ preds = list(zip(*items))[1]
+ refs, preds = _sacreformat(refs, preds)
+ return sacrebleu.corpus_bleu(preds, refs).score
+
+
+@register_aggregation("chrf")
+def chrf(items):
+ """chrF++ is a tool for automatic evaluation of machine translation output
+ based on character n-gram precision and recall enhanced with word n-grams.
+ Source: https://github.com/m-popovic/chrF
+ Paper: https://www.aclweb.org/anthology/W15-3049.pdf
+
+ Higher is better # TODO I think
+ """
+ refs = list(zip(*items))[0]
+ preds = list(zip(*items))[1]
+ refs, preds = _sacreformat(refs, preds)
+ return sacrebleu.corpus_chrf(preds, refs).score
+
+
+@register_aggregation("ter")
+def ter(items):
+ """Translation Error Rate is an error metric for machine translation that
+ measures the number of edits required to change a system output into one
+ of the references
+ Source: http://www.cs.umd.edu/~snover/tercom/
+ Paper: http://mt-archive.info/AMTA-2006-Snover.pdf
+
+ Lower is better
+ """
+ refs = list(zip(*items))[0]
+ preds = list(zip(*items))[1]
+ refs, preds = _sacreformat(refs, preds)
+ return sacrebleu.corpus_ter(preds, refs).score
+
+
+@register_aggregation("brier_score")
+def brier_score(items): # This is a passthrough function
+ gold, predictions = list(zip(*items))
+ bs, num_class = np.array(predictions).shape
+
+ gold = list(gold)
+ gold_one_hot = np.eye(num_class)[gold]
+ return np.mean(np.sum((predictions - gold_one_hot) ** 2, axis=1))
+
+
+@register_metric(
+ metric="brier_score",
+ higher_is_better=False,
+ output_type=["multiple_choice"],
+ aggregation="brier_score",
+)
+def brier_score_fn(items): # This is a passthrough function
+ return items
+
+
+@register_metric(
+ metric="acc",
+ higher_is_better=True,
+ output_type=["loglikelihood", "multiple_choice"],
+ aggregation="mean",
+)
+def acc_fn(items): # This is a passthrough function
+ return items
+
+
+@register_metric(
+ metric="acc_norm",
+ higher_is_better=True,
+ output_type=["loglikelihood", "multiple_choice"],
+ aggregation="mean",
+)
+def acc_norm_fn(items): # This is a passthrough function
+ return items
+
+
+@register_metric(
+ metric="acc_mutual_info",
+ higher_is_better=True,
+ output_type="multiple_choice",
+ aggregation="mean",
+)
+def acc_mutual_info_fn(items): # This is a passthrough function
+ return items
+
+
+### the code used in the `exact_match_hf_evaluate` function is ported from
+### https://github.com/huggingface/evaluate/blob/main/metrics/exact_match/exact_match.py
+### which is under the apache license.
+
+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+def exact_match_hf_evaluate(
+ predictions,
+ references,
+ regexes_to_ignore=None,
+ ignore_case=False,
+ ignore_punctuation=False,
+ ignore_numbers=False,
+):
+ if regexes_to_ignore is not None:
+ for s in regexes_to_ignore:
+ predictions = np.array([re.sub(s, "", x) for x in predictions])
+ references = np.array([re.sub(s, "", x) for x in references])
+ else:
+ predictions = np.asarray(predictions)
+ references = np.asarray(references)
+
+ if ignore_case:
+ predictions = np.char.lower(predictions)
+ references = np.char.lower(references)
+
+ if ignore_punctuation:
+ repl_table = string.punctuation.maketrans("", "", string.punctuation)
+ predictions = np.char.translate(predictions, table=repl_table)
+ references = np.char.translate(references, table=repl_table)
+
+ if ignore_numbers:
+ repl_table = string.digits.maketrans("", "", string.digits)
+ predictions = np.char.translate(predictions, table=repl_table)
+ references = np.char.translate(references, table=repl_table)
+
+ score_list = predictions == references
+
+ return {"exact_match": np.mean(score_list)}
+
+
+###
+
+
+@register_metric(
+ metric="exact_match",
+ higher_is_better=True,
+ output_type="generate_until",
+ aggregation="mean",
+)
+def exact_match_fn(**kwargs):
+ return exact_match_hf_evaluate(**kwargs)
+
+
+@register_metric(
+ metric="perplexity",
+ higher_is_better=False,
+ output_type="loglikelihood",
+ aggregation="perplexity",
+)
+def perplexity_fn(items): # This is a passthrough function
+ return items
+
+
+@register_metric(
+ metric="word_perplexity",
+ higher_is_better=False,
+ output_type="loglikelihood_rolling",
+ aggregation="weighted_perplexity",
+)
+def word_perplexity_fn(items): # This is a passthrough function
+ return items
+
+
+@register_metric(
+ metric="byte_perplexity",
+ higher_is_better=False,
+ output_type="loglikelihood_rolling",
+ aggregation="weighted_perplexity",
+)
+def byte_perplexity_fn(items): # This is a passthrough function
+ return items
+
+
+@register_metric(
+ metric="bits_per_byte",
+ higher_is_better=False,
+ output_type="loglikelihood_rolling",
+ aggregation="bits_per_byte",
+)
+def bits_per_byte_fn(items): # This is a passthrough function
+ return items
+
+
+def levenshtein_distance(s1, s2):
+ if len(s1) > len(s2):
+ s1, s2 = s2, s1
+
+ distances = range(len(s1) + 1)
+ for i2, c2 in enumerate(s2):
+ distances_ = [i2 + 1]
+ for i1, c1 in enumerate(s1):
+ if c1 == c2:
+ distances_.append(distances[i1])
+ else:
+ distances_.append(1 + min((distances[i1], distances[i1 + 1], distances_[-1])))
+ distances = distances_
+ return distances[-1]
+
+
+@register_metric(
+ metric="anls",
+ higher_is_better=True,
+ output_type="generate_until",
+ aggregation="mean",
+)
+def anls(
+ references,
+ predictions,
+ thresh_hold=0.5,
+):
+ """https://github.com/QwenLM/Qwen-VL/blob/master/eval_mm/infographicsvqa_eval.py"""
+ values = []
+ # Unwrap predictions if it's a nested list
+ pred = predictions[0] if isinstance(predictions[0], str) else predictions[0][0]
+
+ for answer in references:
+ # preprocess both the answers - gt and prediction
+ gt_answer = " ".join(answer.strip().lower().split())
+ det_answer = " ".join(pred.strip().lower().split())
+
+ dist = levenshtein_distance(gt_answer, det_answer)
+ length = max(len(answer.upper()), len(pred.upper()))
+ values.append(0.0 if length == 0 else float(dist) / float(length))
+
+ question_result = 1 - min(values)
+
+ if question_result < thresh_hold:
+ question_result = 0
+ return {"anls": question_result}
+
+
+def pop_stddev(arr):
+ mu = mean(arr)
+ return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr))
+
+
+def sample_stddev(arr):
+ mu = mean(arr)
+ return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1))
+
+
+def mean_stderr(arr):
+ return sample_stddev(arr) / math.sqrt(len(arr))
+
+
+@register_metric(
+ metric="bypass",
+ higher_is_better=True,
+ output_type=["loglikelihood", "multiple_choice", "generate_until", "generate_until_multi_round"],
+ aggregation="bypass",
+)
+def bypass(items):
+ return items
+
+
+@register_metric(
+ metric="mcc",
+ higher_is_better=True,
+ output_type="multiple_choice",
+ aggregation="matthews_corrcoef",
+)
+def mcc_fn(items): # This is a passthrough function
+ return items
+
+
+@register_metric(
+ metric="f1",
+ higher_is_better=True,
+ output_type="multiple_choice",
+ aggregation="f1",
+)
+def f1_fn(items): # This is a passthrough function
+ return items
+
+
+@register_metric(
+ metric="bleu",
+ higher_is_better=True,
+ output_type=["generate_until", "generate_until_multi_round"],
+ aggregation="bleu",
+)
+def bleu_fn(items): # This is a passthrough function
+ return items
+
+
+@register_metric(
+ metric="chrf",
+ higher_is_better=True,
+ output_type=["generate_until", "generate_until_multi_round"],
+ aggregation="chrf",
+)
+def chrf_fn(items): # This is a passthrough function
+ return items
+
+
+@register_metric(
+ metric="ter",
+ higher_is_better=True,
+ output_type=["generate_until", "generate_until_multi_round"],
+ aggregation="ter",
+)
+def ter_fn(items): # This is a passthrough function
+ return items
+
+
+@register_metric(
+ metric="acc_all",
+ higher_is_better=True,
+ output_type="loglikelihood",
+ aggregation="mean",
+)
+def acc_all(items):
+ # Only count as correct if all answers are labeled correctly for each question
+ question_scoring_dict = {}
+ preds = list(zip(*items))[0]
+ docs = list(zip(*items))[1]
+
+ for doc, pred in zip(docs, preds):
+ paragraph_id = doc["idx"]["paragraph"]
+ question_id = doc["idx"]["question"]
+ if (paragraph_id, question_id) not in question_scoring_dict:
+ question_scoring_dict[(paragraph_id, question_id)] = []
+
+ gold_label = doc["label"] == 1
+
+ question_scoring_dict[(paragraph_id, question_id)].append(gold_label == pred)
+ acc = np.mean([int(all(x)) for x in question_scoring_dict.values()])
+ return acc
+
+
+def acc_all_stderr(items):
+ # Only count as correct if all answers are labeled correctly for each question
+ question_scoring_dict = {}
+ preds = list(zip(*items))[0]
+ docs = list(zip(*items))[1]
+
+ for doc, pred in zip(docs, preds):
+ question_id = doc["idx"]["question"]
+ if question_id not in question_scoring_dict:
+ question_scoring_dict[question_id] = []
+
+ gold_label = doc["label"] == 1
+ question_scoring_dict[question_id].append(gold_label == pred)
+
+ acc = mean_stderr([int(all(x)) for x in question_scoring_dict.values()])
+ return acc
+
+
+def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
+ """Compute max metric between prediction and each ground truth."""
+ scores_for_ground_truths = []
+ for ground_truth in ground_truths:
+ score = metric_fn(prediction, ground_truth)
+ scores_for_ground_truths.append(score)
+ return max(scores_for_ground_truths)
+
+
+def weighted_mean(items):
+ a, b = zip(*items)
+ return sum(a) / sum(b)
+
+
+def is_non_str_iterable(obj):
+ return isinstance(obj, Iterable) and not isinstance(obj, str)
+
+
+def _sacreformat(refs, preds):
+ """Format refs and preds for sacrebleu corpus calculation. It is very particular"""
+ # Sacrebleu expects (List[str], List[List[str])
+ # e.g. sacrebleu.corpus_bleu([pred_t], [[ref1_stream], [ref2_stream], ...])
+
+ # Note [ref1_stream] is the first reference for each pred.
+ # So lists are size N and (M, N) for N preds and M possible refs for each pred
+ # This is a different order of dimensions that I would expect
+
+ # We expect refs to be List[str] or List[List[str]], the outer list corresponding to preds
+ # Must become List[List[str]] with the inner list corresponding to preds
+ if not is_non_str_iterable(refs):
+ refs = list(refs)
+ if not is_non_str_iterable(refs[0]):
+ refs = [[ref] for ref in refs]
+ refs = list(zip(*refs))
+ # Note the number of refs in each ref list much match the number of preds
+
+ # We expect preds to be List[str] or List[List[str]]. Must become List[str]
+ if not is_non_str_iterable(preds):
+ preds = list(preds)
+ if is_non_str_iterable(preds[0]):
+ assert len(preds[0]) == 1, f"Pred must be a str, was {preds[0]}"
+ preds = [pred[0] for pred in preds]
+
+ return refs, preds
+
+
+# stderr stuff
+
+
+class _bootstrap_internal:
+ def __init__(self, f, n) -> None:
+ self.f = f
+ self.n = n
+
+ def __call__(self, v):
+ i, xs = v
+ rnd = random.Random()
+ rnd.seed(i)
+ res = []
+ for _ in range(self.n):
+ res.append(self.f(rnd.choices(xs, k=len(xs))))
+ return res
+
+
+def bootstrap_stderr(f, xs, iters):
+ import multiprocessing as mp
+
+ pool = mp.Pool(mp.cpu_count())
+ # this gives a biased estimate of the stderr (i.e w/ the mean, it gives something
+ # equivalent to stderr calculated without Bessel's correction in the stddev.
+ # Unfortunately, I haven't been able to figure out what the right correction is
+ # to make the bootstrap unbiased - i considered multiplying by sqrt(n/(n-1)) but
+ # that would be ad-hoc and I can't prove that that would actually be an unbiased estimator)
+ # Thankfully, shouldn't matter because our samples are pretty big usually anyways
+ res = []
+ chunk_size = min(1000, iters)
+ from tqdm import tqdm
+
+ print("bootstrapping for stddev:", f.__name__)
+ for bootstrap in tqdm(
+ pool.imap(
+ _bootstrap_internal(f, chunk_size),
+ [(i, xs) for i in range(iters // chunk_size)],
+ ),
+ total=iters // chunk_size,
+ ):
+ # sample w replacement
+ res.extend(bootstrap)
+
+ pool.close()
+ return sample_stddev(res)
+
+
+def stderr_for_metric(metric, bootstrap_iters: int):
+ if bootstrap_iters <= 0:
+ # return no function (don't compute stderr) if bootstrap iters = 0
+ return None
+
+ bootstrappable = [
+ median,
+ matthews_corrcoef,
+ f1_score,
+ perplexity,
+ bleu,
+ chrf,
+ ter,
+ ]
+
+ if metric in bootstrappable:
+ return lambda x: bootstrap_stderr(metric, x, iters=bootstrap_iters)
+
+ stderr = {mean: mean_stderr, acc_all: acc_all_stderr}
+
+ return stderr.get(metric, None)
+
+
+def pooled_sample_stderr(stderrs: List[float], sizes: List[int]):
+ # Used to aggregate bootstrapped stderrs across subtasks in a group,
+ # when we are weighting by the size of each subtask.
+ #
+
+ assert len(stderrs) == len(sizes)
+
+ # formula source: https://en.wikipedia.org/wiki/Pooled_variance
+ # and: https://stats.stackexchange.com/a/4841331
+ # this empirically seems to match running `stderr_for_metric` on all instances
+ # from the subtasks concatenated with each other.
+ pooled_sample_var = (sum([(size - 1) * stderr**2 * size for size, stderr in zip(sizes, stderrs)])) / (sum(sizes) - len(sizes))
+
+ return np.sqrt(pooled_sample_var / sum(sizes))
+
+
+def combined_sample_stderr(stderrs: List[float], sizes: List[int], metrics=None):
+ assert metrics is not None, "Need to pass a list of each subtask's metric for this stderr aggregation"
+ assert len(stderrs) == len(sizes) and len(sizes) == len(metrics)
+
+ # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1390 for more documentation.
+ # This formula depends on sample means.
+ # removed because it seems to give erroneously huge stderrs for groupings of tasks
+ # and does not seem to match up with bootstrap-calculated stderrs for groups.
+
+ ### don't use this unless a statistician has told you it's the right thing to do ###
+
+ # accumulators: we'll aggregate pairwise N - 1 times
+ variance = stderrs[0] ** 2
+ curr_size = sizes[0]
+ curr_score = metrics[0]
+
+ for stderr, size, score in zip(stderrs[1:], sizes[1:], metrics[1:]):
+ curr_score = ((curr_score * curr_size) + (score * size)) / (curr_size + size) # NOTE: this assumes our aggregation fn is "mean"
+
+ variance = ((curr_size - 1) * variance + (size - 1) * (stderr**2)) / (curr_size + size - 1) + curr_size * size / ((curr_size + size) * (curr_size + size - 1)) * (curr_score - score) ** 2
+
+ return np.sqrt(variance)
+
+
+def aggregate_subtask_metrics(metrics, sizes, weight_by_size=True):
+ # A helper function that is used to aggregate
+ # subtask scores cross-task.
+ # TODO: does not hold for non-mean aggregations
+ if not weight_by_size:
+ sizes = [1] * len(sizes)
+
+ assert len(metrics) == len(sizes)
+
+ return sum([metric * size for metric, size in zip(metrics, sizes)]) / sum(sizes)
diff --git a/lmms-eval/lmms_eval/api/model.py b/lmms-eval/lmms_eval/api/model.py
new file mode 100644
index 0000000000000000000000000000000000000000..8c3de33db6e6d5df83c4016706a994b90efed349
--- /dev/null
+++ b/lmms-eval/lmms_eval/api/model.py
@@ -0,0 +1,234 @@
+import abc
+import gc
+import hashlib
+import json
+import os
+from typing import List, Optional, Tuple, Type, TypeVar, Union
+
+import torch
+import torch.nn as nn
+from loguru import logger as eval_logger
+from sqlitedict import SqliteDict
+from tqdm import tqdm
+
+from lmms_eval import utils
+from lmms_eval.api.instance import Instance
+
+T = TypeVar("T", bound="lmms")
+
+
+class lmms(abc.ABC):
+ is_simple: bool = True
+
+ def __init__(self) -> None:
+ """Defines the interface that should be implemented by all lmms subclasses.
+ lmmss are assumed to take image-text as input and yield strings as output
+ (inputs/outputs should be tokenization-agnostic.)
+ """
+ # set rank and world size to a single process, by default.
+ self._rank = 0
+ self._world_size = 1
+ self.cache_hook = CacheHook(None)
+ self.task_dict = {}
+
+ @abc.abstractmethod
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
+ """Compute log-likelihood of generating a continuation from a context.
+ Downstream tasks should attempt to use loglikelihood instead of other
+ LMM calls whenever possible.
+
+ :param requests: list[Instance]
+ A list of Instance objects, with property `args` which returns a tuple (context, continuation).
+ `context: str`
+ Context string. Implementations of LMM must be able to handle an
+ empty context string.
+ `continuation: str`
+ The continuation over which log likelihood will be calculated. If
+ there is a word boundary, the space should be in the continuation.
+ For example, context="hello" continuation=" world" is correct.
+ 'visual_list: list[dict]'
+ Visual input to the model. Can be None.
+
+ :return: list[tuple[float, bool]]
+ A list of pairs (logprob, isgreedy)
+ `logprob: float`
+ The log probability of `continuation`.
+ `isgreedy`:
+ Whether `continuation` would be generated by greedy sampling from `context`.
+ """
+ pass
+
+ # TODO: Add an optional max length
+ @abc.abstractmethod
+ def generate_until(self, requests) -> List[str]:
+ """Generate greedily until a stopping sequence
+
+ :param requests: list[Instance]
+ A list of Instance objects with property `args` which returns a tuple (context, until).
+ context: str
+ Context string
+ generation_kwargs: dict
+ Generation Kwargs
+ 'visual_list: list[dict]'
+ Visual input to the model. Can be None.
+ :return: list[str]
+ A list of strings continuation
+ continuation: str
+ The generated continuation.
+ """
+ pass
+
+ @abc.abstractmethod
+ def generate_until_multi_round(self, requests) -> List[str]:
+ """Generate greedily until a stopping sequence
+
+ :param requests: list[Instance]
+ A list of Instance objects with property `args` which returns a tuple (context, until).
+ context: str
+ Context string
+ generation_kwargs: dict
+ Generation Kwargs
+ 'visual_list: list[dict]'
+ Visual input to the model. Can be None.
+ :return: list[str]
+ A list of strings continuation
+ continuation: str
+ The generated continuation.
+ """
+ pass
+
+ @classmethod
+ def create_from_arg_string(cls: Type[T], arg_string: str, additional_config: Optional[dict] = None) -> T:
+ """
+ Creates an instance of the LMM class using the given argument string and additional config.
+
+ Parameters:
+ - arg_string: A string containing arguments in the format key1=value1,key2=value2.
+ - additional_config: Optional dictionary containing additional configuration parameters.
+
+ Returns:
+ - Instance of the LMM class.
+ """
+ additional_config = {} if additional_config is None else additional_config
+ args = utils.simple_parse_args_string(arg_string)
+ args2 = {k: v for k, v in additional_config.items() if v is not None}
+ return cls(**args, **args2)
+
+ @property
+ def rank(self):
+ # used in the case of parallelism. Hardcoded to
+ # ensure no errors arise using API models which do
+ # not support multi-device parallelism nor expect it.
+ return self._rank
+
+ @property
+ def world_size(self):
+ # used in the case of parallelism. Hardcoded to
+ # ensure no errors arise using API models which do
+ # not support multi-device parallelism nor expect it.
+ return self._world_size
+
+ def set_cache_hook(self, cache_hook) -> None:
+ self.cache_hook = cache_hook
+
+ def clean(self):
+ for attr_name in list(vars(self)):
+ attr_value = getattr(self, attr_name)
+ if isinstance(attr_value, nn.Module):
+ delattr(self, attr_name)
+ gc.collect()
+ torch.cuda.empty_cache()
+
+
+### SQLite-based caching of LMM responses
+def hash_args(attr, args):
+ dat = json.dumps([attr] + list(args))
+ return hashlib.sha256(dat.encode("utf-8")).hexdigest()
+
+
+class CacheHook:
+ def __init__(self, cachinglm) -> None:
+ if cachinglm is None:
+ self.dbdict = None
+ return
+
+ self.dbdict = cachinglm.dbdict
+
+ def add_partial(self, attr, req, res) -> None:
+ if self.dbdict is None:
+ return
+ hsh = hash_args(attr, req)
+ self.dbdict[hsh] = res
+
+
+class CachingLMM:
+ def __init__(self, lm, cache_db) -> None:
+ """LMM wrapper that returns cached results if they exist, and uses the underlying LMM if not.
+
+ :param lm: LMM
+ Underlying LMM
+ :param cache_db: str
+ Path to cache db
+ """
+ self.lm = lm
+ self.cache_db = cache_db
+ if os.path.dirname(cache_db):
+ os.makedirs(os.path.dirname(cache_db), exist_ok=True)
+ self.dbdict = SqliteDict(cache_db, autocommit=True)
+
+ # add hook to lm
+ lm.set_cache_hook(self.get_cache_hook())
+
+ def __getattr__(self, attr):
+ lm_attr = getattr(self.lm, attr)
+ if not callable(lm_attr):
+ return lm_attr
+
+ def fn(requests):
+ res = []
+ remaining_reqs = []
+ warned = False
+ # figure out which ones are cached and which ones are new
+ eval_logger.info(f"Loading '{attr}' responses from cache '{self.cache_db}' where possible...")
+ for req in tqdm(requests):
+ hsh = hash_args(attr, req.args)
+ if attr in ["generate_until", "generate_until_multi_round"] and req.args[1].get("do_sample", False):
+ # when we are doing non-greedy generation, don't use the cache
+ # (else every "randomly sampled" generation would be identical for repeats > 1).
+ if not warned:
+ eval_logger.warning(f"Arguments to lm.generate_until() '{req.args[1]}' include non-deterministic sampling. Caching will not be performed for such requests.")
+ warned = True
+ res.append(None)
+ remaining_reqs.append(req)
+ elif hsh in self.dbdict:
+ ob = self.dbdict[hsh]
+
+ assert ob is not None
+
+ res.append(ob)
+ else:
+ res.append(None)
+ remaining_reqs.append(req)
+
+ # actually run the LMM on the requests that do not have cached results
+ rem_res = getattr(self.lm, attr)(remaining_reqs)
+
+ # stick the new ones back into the list and also cache any of the new ones
+ resptr = 0
+ for req, r in zip(remaining_reqs, rem_res):
+ while res[resptr] is not None:
+ resptr += 1
+
+ res[resptr] = r
+
+ # caching
+ hsh = hash_args(attr, req.args)
+ self.dbdict[hsh] = r
+ self.dbdict.commit()
+
+ return res
+
+ return fn
+
+ def get_cache_hook(self):
+ return CacheHook(self)
diff --git a/lmms-eval/lmms_eval/api/registry.py b/lmms-eval/lmms_eval/api/registry.py
new file mode 100644
index 0000000000000000000000000000000000000000..c82a8ba6127bd915ad7a63a726fb11aa01808a87
--- /dev/null
+++ b/lmms-eval/lmms_eval/api/registry.py
@@ -0,0 +1,185 @@
+from typing import Callable, Dict, Union
+
+import evaluate as hf_evaluate
+from loguru import logger as eval_logger
+
+from lmms_eval.api.model import lmms
+
+MODEL_REGISTRY = {}
+
+
+def register_model(*names):
+ # either pass a list or a single alias.
+ # function receives them as a tuple of strings
+
+ def decorate(cls):
+ for name in names:
+ assert issubclass(cls, lmms), f"Model '{name}' ({cls.__name__}) must extend lmms class"
+
+ assert name not in MODEL_REGISTRY, f"Model named '{name}' conflicts with existing model! Please register with a non-conflicting alias instead."
+
+ MODEL_REGISTRY[name] = cls
+ return cls
+
+ return decorate
+
+
+def get_model(model_name):
+ try:
+ return MODEL_REGISTRY[model_name]
+ except KeyError:
+ raise ValueError(f"Attempted to load model '{model_name}', but no model for this name found! Supported model names: {', '.join(MODEL_REGISTRY.keys())}")
+
+
+TASK_REGISTRY = {} # Key: task name, Value: task ConfigurableTask class
+GROUP_REGISTRY = {} # Key: group name, Value: list of task names or group names
+TASK_INITIALIZED = False
+ALL_TASKS = set() # Set of all task names and group names
+func2task_index = {} # Key: task ConfigurableTask class, Value: task name
+OUTPUT_TYPE_REGISTRY = {}
+METRIC_REGISTRY = {}
+METRIC_AGGREGATION_REGISTRY = {}
+AGGREGATION_REGISTRY: Dict[str, Callable[[], Dict[str, Callable]]] = {}
+HIGHER_IS_BETTER_REGISTRY = {}
+FILTER_REGISTRY = {}
+
+
+def register_task(name):
+ def decorate(fn):
+ assert name not in TASK_REGISTRY, f"task named '{name}' conflicts with existing registered task!"
+
+ TASK_REGISTRY[name] = fn
+ ALL_TASKS.add(name)
+ func2task_index[fn.__name__] = name
+ return fn
+
+ return decorate
+
+
+def register_group(name):
+ def decorate(fn):
+ func_name = func2task_index[fn.__name__]
+ if name in GROUP_REGISTRY:
+ GROUP_REGISTRY[name].append(func_name)
+ else:
+ GROUP_REGISTRY[name] = [func_name]
+ ALL_TASKS.add(name)
+ return fn
+
+ return decorate
+
+
+OUTPUT_TYPE_REGISTRY = {}
+METRIC_REGISTRY = {}
+METRIC_AGGREGATION_REGISTRY = {}
+AGGREGATION_REGISTRY = {}
+HIGHER_IS_BETTER_REGISTRY = {}
+
+DEFAULT_METRIC_REGISTRY = {
+ "loglikelihood": [
+ "perplexity",
+ "acc",
+ ],
+ "multiple_choice": ["acc", "acc_norm"],
+ "generate_until": ["exact_match"],
+ "generate_until_multi_round": ["exact_match"],
+}
+
+
+def register_metric(**args):
+ # TODO: do we want to enforce a certain interface to registered metrics?
+ def decorate(fn):
+ assert "metric" in args
+ name = args["metric"]
+
+ for key, registry in [
+ ("metric", METRIC_REGISTRY),
+ ("higher_is_better", HIGHER_IS_BETTER_REGISTRY),
+ ("aggregation", METRIC_AGGREGATION_REGISTRY),
+ ]:
+ if key in args:
+ value = args[key]
+ assert value not in registry, f"{key} named '{value}' conflicts with existing registered {key}!"
+
+ if key == "metric":
+ registry[name] = fn
+ elif key == "aggregation":
+ registry[name] = AGGREGATION_REGISTRY[value]
+ else:
+ registry[name] = value
+
+ return fn
+
+ return decorate
+
+
+def get_metric(name: str, hf_evaluate_metric=False) -> Callable:
+ if not hf_evaluate_metric:
+ if name in METRIC_REGISTRY:
+ return METRIC_REGISTRY[name]
+ else:
+ eval_logger.warning(f"Could not find registered metric '{name}' in lm-eval, searching in HF Evaluate library...")
+
+ try:
+ metric_object = hf_evaluate.load(name)
+ return metric_object.compute
+ except Exception:
+ eval_logger.error(
+ f"{name} not found in the evaluate library! Please check https://huggingface.co/evaluate-metric",
+ )
+
+
+def register_aggregation(name):
+ def decorate(fn):
+ assert name not in AGGREGATION_REGISTRY, f"aggregation named '{name}' conflicts with existing registered aggregation!"
+
+ AGGREGATION_REGISTRY[name] = fn
+ return fn
+
+ return decorate
+
+
+def get_aggregation(name):
+ try:
+ return AGGREGATION_REGISTRY[name]
+ except KeyError:
+ eval_logger.warning(
+ "{} not a registered aggregation metric!".format(name),
+ )
+
+
+def get_metric_aggregation(name):
+ try:
+ return METRIC_AGGREGATION_REGISTRY[name]
+ except KeyError:
+ eval_logger.warning(
+ "{} metric is not assigned a default aggregation!".format(name),
+ )
+
+
+def is_higher_better(metric_name):
+ try:
+ return HIGHER_IS_BETTER_REGISTRY[metric_name]
+ except KeyError:
+ eval_logger.warning(f"higher_is_better not specified for metric '{metric_name}'!")
+
+
+def register_filter(name):
+ def decorate(cls):
+ if name in FILTER_REGISTRY:
+ eval_logger.info(f"Registering filter `{name}` that is already in Registry {FILTER_REGISTRY}")
+ FILTER_REGISTRY[name] = cls
+ return cls
+
+ return decorate
+
+
+def get_filter(filter_name: Union[str, Callable]) -> Callable:
+ try:
+ return FILTER_REGISTRY[filter_name]
+ except KeyError as e:
+ if callable(filter_name):
+ return filter_name
+ else:
+ eval_logger.warning(f"filter `{filter_name}` is not registered!")
+ raise e
diff --git a/lmms-eval/lmms_eval/api/samplers.py b/lmms-eval/lmms_eval/api/samplers.py
new file mode 100644
index 0000000000000000000000000000000000000000..f77065e8e53b377fcf30acbc91fcc4bc2f05f9fd
--- /dev/null
+++ b/lmms-eval/lmms_eval/api/samplers.py
@@ -0,0 +1,94 @@
+class ContextSampler:
+ def __init__(self, docs, task, fewshot_indices=None, rnd=None) -> None:
+ self.rnd = rnd
+ assert self.rnd, "must pass rnd to FewShotSampler!"
+
+ self.task = task
+ self.config = task._config
+
+ self.target_delimiter = self.config.target_delimiter
+ self.fewshot_delimiter = self.config.fewshot_delimiter
+
+ self.doc_to_text = self.task.doc_to_text
+ self.doc_to_target = self.task.doc_to_target
+ self.doc_to_choice = self.task.doc_to_choice
+
+ self.docs = docs # HF dataset split, provided by task._fewshot_docs()
+ if fewshot_indices: # subset few-shot docs from
+ self.docs = self.docs.select(fewshot_indices)
+
+ def get_context(self, doc, num_fewshot):
+ # draw an extra fewshot sample if using same split as evaluating on
+ n_samples = num_fewshot + 1 if self.config.fewshot_split == self.config.test_split else num_fewshot
+
+ # draw `n_samples` docs from fewshot_docs
+ fewshotex = self.sample(n_samples)
+
+ # get rid of the doc that's the one we're evaluating, if it's in the fewshot
+ # TODO: should we just stop people from using fewshot from same split as evaluating?
+ selected_docs = [x for x in fewshotex if x != doc][:num_fewshot]
+
+ labeled_examples = (
+ self.fewshot_delimiter.join(
+ [
+ # TODO: is separating doc_to_text and doc_to_target by one space always desired?
+ (self.doc_to_text(doc) if (self.config.doc_to_choice is None or type(self.doc_to_text(doc)) is str) else self.doc_to_choice(doc)[self.doc_to_text(doc)])
+ + self.target_delimiter
+ + (
+ str(self.doc_to_target(doc)[0])
+ if type(self.doc_to_target(doc)) is list
+ else self.doc_to_target(doc) if (self.config.doc_to_choice is None or type(self.doc_to_target(doc)) is str) else str(self.doc_to_choice(doc)[self.doc_to_target(doc)])
+ )
+ for doc in selected_docs
+ ]
+ )
+ + self.fewshot_delimiter
+ )
+
+ return labeled_examples
+
+ def sample(self, n):
+ """
+ Draw `n` samples from our fewshot docs. This method should be overridden by subclasses.
+ """
+
+ return self.rnd.sample(self.docs, n)
+
+
+class FirstNSampler(ContextSampler):
+ def sample(self, n) -> None:
+ """
+ Draw the first `n` samples in order from the specified split.
+ Used for tasks with "canonical" ordered fewshot examples, such as MMLU and CMMLU.
+ """
+ assert n <= len(self.docs), f"Error: number of fewshot samples requested exceeds the {len(self.docs)} that are available."
+ return self.docs[:n]
+
+
+class BalancedSampler(ContextSampler):
+ def sample(self, n) -> None:
+ """
+ TODO: this should return approximately class-balanced samples from our fewshot examples.
+ TODO: what order should they be in? maybe random?
+ """
+
+ pass
+
+
+class ManualSampler(ContextSampler):
+ def sample(self, n) -> None:
+ """ """
+ pass
+
+
+SAMPLER_REGISTRY = {
+ "default": ContextSampler,
+ "first_n": FirstNSampler,
+}
+
+
+def get_sampler(name):
+ try:
+ return SAMPLER_REGISTRY[name]
+ except KeyError:
+ raise ValueError(f"Attempted to use contextsampler '{name}', but no sampling strategy for this name found! Supported model names: {', '.join(SAMPLER_REGISTRY.keys())}")
diff --git a/lmms-eval/lmms_eval/api/task.py b/lmms-eval/lmms_eval/api/task.py
new file mode 100644
index 0000000000000000000000000000000000000000..de5d3ddd5daf1fd06c3d28f0912da42d815cb8ff
--- /dev/null
+++ b/lmms-eval/lmms_eval/api/task.py
@@ -0,0 +1,1687 @@
+import abc
+import ast
+import copy
+import inspect
+import itertools
+import json
+import os
+import random
+import re
+import shutil
+import subprocess
+from collections.abc import Callable
+from dataclasses import asdict, dataclass, field
+from functools import partial
+from glob import glob
+from typing import (
+ Any,
+ Dict,
+ Iterable,
+ Iterator,
+ List,
+ Literal,
+ Mapping,
+ Optional,
+ Tuple,
+ Union,
+)
+
+import datasets
+import numpy as np
+from accelerate import Accelerator
+from datasets import Audio, DownloadConfig, Image, Sequence
+from huggingface_hub import snapshot_download
+from loguru import logger as eval_logger
+from PIL import Image as PIL_Image
+from PIL import ImageFile
+from tenacity import retry, stop_after_attempt, stop_after_delay, wait_fixed
+from tqdm import tqdm
+
+from lmms_eval import utils
+from lmms_eval.api import samplers
+from lmms_eval.api.instance import Instance
+from lmms_eval.api.registry import (
+ AGGREGATION_REGISTRY,
+ DEFAULT_METRIC_REGISTRY,
+ METRIC_REGISTRY,
+ OUTPUT_TYPE_REGISTRY,
+ get_aggregation,
+ get_metric,
+ get_metric_aggregation,
+ is_higher_better,
+)
+from lmms_eval.caching.cache import load_from_cache, save_to_cache
+from lmms_eval.filters import build_filter_ensemble
+
+# HuggingfaceM4/NoCaps contains truncated image in test split
+# Include this inside code block to avoid error
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+ALL_OUTPUT_TYPES = [
+ "loglikelihood",
+ "multiple_choice",
+ "generate_until",
+ "generate_until_multi_round",
+]
+
+
+@dataclass
+class TaskConfig(dict):
+ # task naming/registry
+ task: str = None
+ task_alias: str = None
+ tag: str = None
+ group: Union[str, list] = None
+ group_alias: Union[str, list] = None
+ # HF dataset options.
+ # which dataset to use,
+ # and what splits for what purpose
+ dataset_path: str = None
+ dataset_name: str = None
+ dataset_kwargs: dict = None
+ training_split: str = None
+ validation_split: str = None
+ test_split: str = None
+ fewshot_split: str = None # TODO: assert that this not None if num_fewshot > 0. (?) assert if this is same split as one evaling (?)
+ full_docs: bool = False
+ # formatting / prompting options.
+ # see docs/advanced_task_guide.md for more info
+ process_results_use_image: bool = False
+ process_docs: Callable = None
+ doc_to_visual: Union[Callable, str] = None
+ doc_to_text: Union[Callable, str] = None
+ doc_to_target: Union[Callable, str] = None
+ doc_to_choice: Union[Callable, str, dict, list] = None
+ doc_to_messages: Callable = None
+ process_results: Union[Callable, str] = None
+ use_prompt: str = None
+ description: str = ""
+ target_delimiter: str = " "
+ fewshot_delimiter: str = "\n\n"
+ fewshot_config: dict = None
+ # runtime configuration options
+ num_fewshot: int = None
+ # scoring options
+ metric_list: list = None
+ output_type: str = "generate_until"
+ generation_kwargs: dict = None
+ repeats: int = 1
+ filter_list: Union[str, list] = None
+ should_decontaminate: bool = False
+ doc_to_decontamination_query: str = None
+
+ metadata: Union[str, list] = None # by default, not used in the code. allows for users to pass arbitrary info to tasks
+
+ lmms_eval_specific_kwargs: dict = None
+ model_specific_generation_kwargs: dict = None
+ model_specific_target_kwargs: dict = None
+
+ def __post_init__(self) -> None:
+ if self.dataset_path and os.path.exists(os.path.dirname(self.dataset_path)):
+ import inspect
+ from importlib import import_module
+
+ # self.dataset_path = inspect.getfile(import_module(self.dataset_path))
+
+ if self.group is not None:
+ eval_logger.warning(
+ "A task YAML file was found to contain a `group` key. Groups which provide aggregate scores over several subtasks now require a separate config file--if not aggregating, you may want to use the `tag` config option instead within your config. Setting `group` within a TaskConfig will be deprecated in v0.4.4. Please see https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/task_guide.md for more information."
+ )
+
+ if self.tag is None:
+ self.tag = self.group
+ else:
+ raise ValueError("Got both a `group` and `tag` entry within a TaskConfig. Please use one or the other--`group` values will be deprecated in v0.4.4.")
+
+ if self.generation_kwargs is not None:
+ if "generate_until" not in self.output_type:
+ eval_logger.warning(f"[{self.task}] passed `generation_kwargs`, but not using `output_type: generate_until`!")
+ assert "generate_until" not in self.output_type
+
+ if "temperature" in self.generation_kwargs:
+ self.generation_kwargs["temperature"] = float(self.generation_kwargs["temperature"])
+
+ if "until" not in self.generation_kwargs:
+ self.generation_kwargs["until"] = [self.fewshot_delimiter]
+ else:
+ if "generate_until" in self.output_type:
+ # ensure that we greedily generate in absence of explicit arguments otherwise
+ self.generation_kwargs = {
+ "until": None if self.fewshot_delimiter is None else [self.fewshot_delimiter],
+ "do_sample": False,
+ }
+
+ # TODO: how to make TaskConfigs be de- and re-serializable, even when using the !function constructor?
+
+ def __getitem__(self, item):
+ return getattr(self, item)
+
+ def __setitem__(self, item, value):
+ return setattr(self, item, value)
+
+ def to_dict(self):
+ """dumps the current config as a dictionary object, as a printable format.
+ null fields will not be printed.
+ Used for dumping results alongside full task configuration
+
+ :return: dict
+ A printable dictionary version of the TaskConfig object.
+
+ # TODO: should any default value in the TaskConfig not be printed?
+ """
+ cfg_dict = asdict(self)
+ # remove values that are `None`
+ for k, v in list(cfg_dict.items()):
+ if v is None:
+ cfg_dict.pop(k)
+ elif isinstance(v, Callable):
+ # TODO: this should handle Promptsource template objects as a separate case?
+ cfg_dict[k] = str(v)
+ return cfg_dict
+
+
+class Task(abc.ABC):
+ """A task represents an entire benchmark including its dataset, problems,
+ answers, and evaluation methods. See MME for a simple example implementation
+
+ A `doc` can be any python object which represents one instance of evaluation.
+ This is usually a dictionary e.g.
+ {"question": ..., "answer": ...} or
+ {"question": ..., question, answer)
+ """
+
+ VERSION = None
+
+ # The name of the `Task` benchmark as denoted in the HuggingFace datasets Hub
+ # or a path to a custom `datasets` loading script.
+ DATASET_PATH: str = None
+
+ # The name of a subset within `DATASET_PATH`.
+ DATASET_NAME: str = None
+
+ OUTPUT_TYPE: str = None
+
+ def __init__(
+ self,
+ data_dir=None,
+ cache_dir=None,
+ download_mode=None,
+ config=None,
+ ) -> None:
+ """
+ :param data_dir: str
+ Stores the path to a local folder containing the `Task`'s data files.
+ Use this to specify the path to manually downloaded data (usually when
+ the dataset is not publicly accessible).
+ :param cache_dir: str
+ The directory to read/write the `Task` dataset. This follows the
+ HuggingFace `datasets` API with the default cache directory located at:
+ `~/.cache/huggingface/datasets`
+ NOTE: You can change the cache location globally for a given process
+ to another directory:
+ `export HF_DATASETS_CACHE="/path/to/another/directory"`
+ :param download_mode: datasets.DownloadMode
+ How to treat pre-existing `Task` downloads and data.
+ - `datasets.DownloadMode.REUSE_DATASET_IF_EXISTS`
+ Reuse download and reuse dataset.
+ - `datasets.DownloadMode.REUSE_CACHE_IF_EXISTS`
+ Reuse download with fresh dataset.
+ - `datasets.DownloadMode.FORCE_REDOWNLOAD`
+ Fresh download and fresh dataset.
+ """
+ self.download(data_dir, cache_dir, download_mode)
+ self._training_docs = None
+ self._fewshot_docs = None
+ self._instances = None
+
+ self._config = TaskConfig({**config}) if config else TaskConfig()
+
+ self._filters = [build_filter_ensemble("none", [["take_first", None]])]
+
+ def download(self, data_dir=None, cache_dir=None, download_mode=None) -> None:
+ """Downloads and returns the task dataset.
+ Override this method to download the dataset from a custom API.
+
+ :param data_dir: str
+ Stores the path to a local folder containing the `Task`'s data files.
+ Use this to specify the path to manually downloaded data (usually when
+ the dataset is not publicly accessible).
+ :param cache_dir: str
+ The directory to read/write the `Task` dataset. This follows the
+ HuggingFace `datasets` API with the default cache directory located at:
+ `~/.cache/huggingface/datasets`
+ NOTE: You can change the cache location globally for a given process
+ by setting the shell environment variable, `HF_DATASETS_CACHE`,
+ to another directory:
+ `export HF_DATASETS_CACHE="/path/to/another/directory"`
+ :param download_mode: datasets.DownloadMode
+ How to treat pre-existing `Task` downloads and data.
+ - `datasets.DownloadMode.REUSE_DATASET_IF_EXISTS`
+ Reuse download and reuse dataset.
+ - `datasets.DownloadMode.REUSE_CACHE_IF_EXISTS`
+ Reuse download with fresh dataset.
+ - `datasets.DownloadMode.FORCE_REDOWNLOAD`
+ Fresh download and fresh dataset.
+ """
+ self.dataset = datasets.load_dataset(
+ path=self.DATASET_PATH,
+ name=self.DATASET_NAME,
+ data_dir=data_dir,
+ cache_dir=cache_dir,
+ download_mode=download_mode,
+ )
+ self.dataset_no_image = datasets.load_dataset(
+ path=self.DATASET_PATH,
+ name=self.DATASET_NAME,
+ data_dir=data_dir,
+ cache_dir=cache_dir,
+ download_mode=download_mode,
+ )
+ for doc_name in self.dataset_no_image:
+ remove_cols = []
+ features = self.dataset_no_image[doc_name].features
+ # If it is an Image instance or a Sequence of Image instance. Remove it
+ for feature in features:
+ if isinstance(features[feature], Image):
+ remove_cols.append(feature)
+ elif isinstance(features[feature], Sequence) and isinstance(features[feature].feature, Image):
+ remove_cols.append(feature)
+ for remove_col in remove_cols:
+ self.dataset_no_image[doc_name] = self.dataset_no_image[doc_name].remove_columns(remove_col)
+
+ @property
+ def config(self):
+ """Returns the TaskConfig associated with this class."""
+ return self._config
+
+ @abc.abstractmethod
+ def has_training_docs(self):
+ """Whether the task has a training set"""
+ pass
+
+ @abc.abstractmethod
+ def has_validation_docs(self):
+ """Whether the task has a validation set"""
+ pass
+
+ @abc.abstractmethod
+ def has_test_docs(self):
+ """Whether the task has a test set"""
+ pass
+
+ def training_docs(self):
+ """
+ :return: Iterable[obj]
+ A iterable of any object, that doc_to_text can handle
+ """
+ return []
+
+ def validation_docs(self):
+ """
+ :return: Iterable[obj]
+ A iterable of any object, that doc_to_text can handle
+ """
+ return []
+
+ def test_docs(self):
+ """
+ :return: Iterable[obj]
+ A iterable of any object, that doc_to_text can handle
+ """
+ return []
+
+ def fewshot_docs(self):
+ """
+ :return: Iterable[obj]
+ A iterable of any object, that doc_to_text can handle
+ """
+ if self.has_training_docs():
+ return self.training_docs()
+ elif self.has_validation_docs():
+ return self.validation_docs()
+ else:
+ if self.config.num_fewshot is not None:
+ eval_logger.warning("has_training_docs and has_validation_docs are False" ", using test_docs as fewshot_docs but this is not recommended.")
+ return self.test_docs()
+
+ def _process_doc(self, doc):
+ """
+ Override this to process (detokenize, strip, replace, etc.) individual
+ documents. This can be used in a map over documents of a data split.
+ E.g. `map(self._process_doc, self.dataset["validation"])`
+
+ :return: dict
+ The processed version of the specified `doc`.
+ """
+ return doc
+
+ @property
+ def instances(self):
+ """After calling `task.build_all_requests()`, tasks
+ maintain a list of the dataset instances which will be evaluated.
+ """
+ return self._instances
+
+ def fewshot_examples(self, k, rnd):
+ if self._training_docs is None:
+ self._training_docs = list(self.training_docs())
+
+ return rnd.sample(self._training_docs, k)
+
+ def doc_to_decontamination_query(self, doc) -> None:
+ print("Override doc_to_decontamination_query with document specific decontamination query.")
+ assert False
+
+ @abc.abstractmethod
+ def doc_to_text(self, doc):
+ pass
+
+ @abc.abstractmethod
+ def doc_to_target(self, doc):
+ pass
+
+ # @profile
+ def build_all_requests(
+ self,
+ *,
+ limit: Union[int, None] = None,
+ rank: int = 0,
+ world_size: int = 1,
+ cache_requests: bool = False,
+ rewrite_requests_cache: bool = False,
+ system_instruction: Optional[str] = None,
+ apply_chat_template: bool = False,
+ fewshot_as_multiturn: bool = False,
+ chat_template: Optional[Callable] = None,
+ tokenizer_name: str = "",
+ ) -> None:
+ """Build a set of Instances for a task, and store them in task.instances"""
+ if self.has_test_docs():
+ docs = self.test_docs()
+ split = self.config.test_split
+ elif self.has_validation_docs():
+ docs = self.validation_docs()
+ split = self.config.validation_split
+ else:
+ assert False, f"Task dataset (path={self.DATASET_PATH}, name={self.DATASET_NAME}) must have valid or test docs!"
+
+ # used with caching
+ og_limit = limit
+
+ cache_key = f"requests-{self._config.task}-{self.config.num_fewshot}shot-rank{rank}-world_size{world_size}"
+ cache_key += "-chat_template" if apply_chat_template else ""
+ cache_key += "-fewshot_as_multiturn" if fewshot_as_multiturn else ""
+ cache_key += f"-system_prompt_hash{utils.hash_string(system_instruction)}" if system_instruction is not None else ""
+ cache_key += f"-tokenizer{tokenizer_name}"
+
+ cached_instances = load_from_cache(file_name=cache_key)
+
+ if cache_requests and cached_instances and not rewrite_requests_cache:
+ cached_instances = cached_instances[:limit]
+
+ flattened_instances = [instance for instance_group in cached_instances for instance in instance_group]
+
+ self._instances = flattened_instances
+ return
+
+ eval_logger.info(f"Building contexts for {self.config.task} on rank {rank}...")
+
+ instances = []
+
+ # process all documents when caching is specified for simplicity
+ if cache_requests and (not cached_instances or rewrite_requests_cache) and limit is not None:
+ limit = None
+
+ doc_id_docs = utils.create_iterator(enumerate(self.eval_docs_no_media), rank=rank, limit=int(limit) if limit else None, world_size=world_size)
+ doc_iterator_for_counting = itertools.islice(range(len(self.test_docs())), rank, limit, world_size) if self.has_test_docs() else itertools.islice(range(len(self.validation_docs())), rank, limit, world_size)
+
+ num_docs = sum(1 for _ in doc_iterator_for_counting)
+
+ for doc_id, doc in tqdm(
+ doc_id_docs,
+ total=num_docs,
+ ):
+ # sample fewshot context #TODO: need to offset doc_id by rank now!
+ fewshot_ctx = self.fewshot_context(
+ doc,
+ 0 if self.config.num_fewshot is None else self.config.num_fewshot,
+ system_instruction,
+ apply_chat_template,
+ fewshot_as_multiturn,
+ chat_template,
+ )
+
+ # TODO: we should override self.config.repeats if doing greedy gen so users don't waste time+compute
+ per_task_metadata = {"task": self.config["task"], "doc_id": doc_id, "repeats": self.config.repeats, "split": split}
+ if self.config.metadata and type(self.config.metadata) == dict: # TODO: temporary fix for metadata loading, ignore the list of dict type.
+ per_task_metadata.update(self.config.metadata)
+
+ inst = self.construct_requests(doc_id=doc_id, ctx=fewshot_ctx, metadata=per_task_metadata)
+
+ if not isinstance(inst, list):
+ inst = [inst]
+
+ instances.append(inst)
+
+ # now flatten, this is to allow slicing to work with pickles
+
+ sliced_instances = instances[:og_limit]
+
+ flattened_instances = [instance for instance_group in sliced_instances for instance in instance_group]
+
+ self._instances = flattened_instances
+
+ if len(self._instances) == 0:
+ raise ValueError("task.build_requests() did not find any docs!")
+
+ if cache_requests and (not cached_instances or rewrite_requests_cache):
+ save_to_cache(file_name=cache_key, obj=instances)
+
+ # FIXME: Bo - We need to check if the doc_to_visual if it's exists and restore it. If we use cache, the doc_to_visual will be None since it's not serializable
+ for instance in self._instances:
+ if instance.arguments[2] is None:
+ arguments = (instance.arguments[0], instance.arguments[1], self.doc_to_visual, *instance.arguments[3:])
+ else:
+ arguments = instance.arguments
+
+ instance.arguments = arguments
+
+ @abc.abstractmethod
+ def construct_requests(self, doc_id, ctx, **kwargs):
+ """Uses RequestFactory to construct Requests and returns an iterable of
+ Requests which will be sent to the LMM.
+
+ :param doc_id: int
+ The index of a document within `self.test_docs()` or `self.validation_docs()`,
+ whichever is the main split used.
+ :param ctx: str
+ The context string, generated by fewshot_context. This includes the natural
+ language description, as well as the few shot examples, and the question
+ part of the document for `doc`.
+ :param repeats: int
+ TODO: update this docstring
+ The number of times each instance in a dataset is inferred on. Defaults to 1,
+ can be increased for techniques like majority voting.
+ """
+ pass
+
+ @abc.abstractmethod
+ def process_results(self, doc, results):
+ """Take a single document and the LMM results and evaluates, returning a
+ dict where keys are the names of submetrics and values are the values of
+ the metric for that one document
+
+ :param doc:
+ The document as returned from training_docs, validation_docs, or test_docs.
+ :param results:
+ The results of the requests created in construct_requests.
+ """
+ pass
+
+ @abc.abstractmethod
+ def aggregation(self):
+ """
+ :returns: {str: [metric_score] -> float}
+ A dictionary where keys are the names of submetrics and values are
+ functions that aggregate a list of metric scores
+ """
+ pass
+
+ @abc.abstractmethod
+ def higher_is_better(self):
+ """
+ :returns: {str: bool}
+ A dictionary where keys are the names of submetrics and values are
+ whether a higher value of the submetric is better
+ """
+ pass
+
+ @classmethod
+ def count_bytes(cls, doc):
+ """Used for byte-level perplexity metrics in rolling loglikelihood"""
+ return len(doc.encode("utf-8"))
+
+ @utils.positional_deprecated
+ def fewshot_context(
+ self,
+ doc_id,
+ num_fewshot,
+ split,
+ rnd=random.Random(1234),
+ description=None,
+ ):
+ """Returns a fewshot context string that is made up of a prepended description
+ (if provided), the `num_fewshot` number of examples, and an appended prompt example.
+
+ :param doc_id: int
+ The document id as returned from training_docs, validation_docs, or test_docs.
+ :param num_fewshot: int
+ The number of fewshot examples to provide in the returned context string.
+ :param split: str
+ The split of the document to retrieve from the dataset
+ :param rnd: random.Random
+ The pseudo-random number generator used to randomly sample examples.
+ WARNING: This is currently a required arg although it's optionalized with a default `None`.
+ :param description: str
+ The task's description that will be prepended to the fewshot examples.
+ :returns: str
+ The fewshot context.
+ """
+ assert rnd is not None, "A `random.Random` generator argument must be provided to `rnd`"
+
+ description = description if description else ""
+ doc = self.dataset_no_image[split][doc_id]
+
+ if num_fewshot == 0:
+ labeled_examples = ""
+ else:
+ # for sets with no training docs, draw from other set *but ensure no overlap with current doc*
+ if self.has_training_docs():
+ fewshotex = self.fewshot_examples(k=num_fewshot, rnd=rnd)
+ else:
+ if self._fewshot_docs is None:
+ self._fewshot_docs = list(self.validation_docs() if self.has_validation_docs() else self.test_docs())
+
+ fewshotex = rnd.sample(self._fewshot_docs, num_fewshot + 1)
+
+ # get rid of the doc that's the one we're evaluating, if it's in the fewshot
+ fewshotex = [x for x in fewshotex if x != doc][:num_fewshot]
+
+ labeled_examples = "\n\n".join([self.doc_to_text(doc) + self.doc_to_target(doc) for doc in fewshotex]) + "\n\n"
+
+ example = self.doc_to_text(doc)
+ return description + labeled_examples + example
+
+ def apply_filters(self) -> Optional[List[Instance]]:
+ """Iterates over FilterEnsembles and applies them to instances"""
+ if hasattr(self, "_filters"):
+ for f in self._filters:
+ f.apply(self._instances, None)
+ else:
+ eval_logger.warning("No filter defined, passing through instances")
+ return self._instances
+
+ def dump_config(self) -> dict:
+ """Returns a dictionary representing the task's config.
+
+ :returns: str
+ The fewshot context.
+ """
+ # TODO: this should only return the overrides applied to a non-YAML task's configuration.
+ # (num_fewshot)
+ return self.config.to_dict()
+
+ def set_config(self, key: str, value: Any, update: bool = False) -> None:
+ """Set or update the configuration for a given key."""
+ if key is None:
+ raise ValueError("Key must be provided.")
+
+ if update:
+ current_value = getattr(self._config, key, {})
+ if not isinstance(current_value, dict):
+ raise TypeError(f"Expected a dict for key '{key}', got {type(current_value).__name__} instead.")
+ current_value.update(value)
+ else:
+ setattr(self._config, key, value)
+
+ def override_metric(self, metric_name: str) -> None:
+ """
+ Override the default metrics used for evaluation with custom metrics.
+
+ Parameters:
+ - metric_name (str): The name of the custom metric to override. Should be registered in api.metrics.
+ """
+ (
+ self._metric_fn_list,
+ self._aggregation_list,
+ self._metric_fn_kwargs,
+ self._higher_is_better,
+ ) = ({}, {}, {}, {})
+ self._metric_fn_list[metric_name] = get_metric(metric_name)
+ self._aggregation_list[metric_name] = get_metric_aggregation(metric_name)
+ self._higher_is_better[metric_name] = is_higher_better(metric_name)
+ self._metric_fn_kwargs[metric_name] = {}
+ if not isinstance(self, ConfigurableTask):
+ self.process_results = lambda x, y: {metric_name: get_metric(metric_name)}
+ self.aggregation = lambda: {metric_name: get_metric_aggregation(metric_name)}
+ setattr(self._config, "metric_list", [{"metric": metric_name}])
+ setattr(self._config, "process_results", None)
+
+ def set_fewshot_seed(self, seed: Optional[int] = None) -> None:
+ self.fewshot_rnd = random.Random(seed)
+ if hasattr(self, "sampler"):
+ self.sampler.rnd = self.fewshot_rnd
+
+ @property
+ def eval_docs(self) -> Union[datasets.Dataset, List[dict]]:
+ if self.has_test_docs():
+ return self.test_docs()
+ elif self.has_validation_docs():
+ return self.validation_docs()
+ else:
+ raise ValueError(f"Task dataset (path={self.DATASET_PATH}, name={self.DATASET_NAME}) must have valid or test docs!")
+
+ def doc_iterator(self, *, rank: int = 0, limit: Union[int, None] = None, world_size: int = 1) -> Iterator[Tuple[int, Any]]:
+ limit = int(limit) if limit else None
+ doc_iterator = utils.create_iterator(
+ enumerate(self.eval_docs),
+ rank=int(rank),
+ limit=limit,
+ world_size=int(world_size),
+ )
+ return doc_iterator
+
+
+class ConfigurableTask(Task):
+ VERSION = "Yaml"
+ OUTPUT_TYPE = None
+ CONFIG = None
+
+ def __init__(
+ self,
+ data_dir=None,
+ cache_dir=None,
+ download_mode=None,
+ config: Optional[dict] = None,
+ model_name: Optional[str] = None,
+ ) -> None: # TODO no super() call here
+ # Get pre-configured attributes
+ self._config = self.CONFIG
+
+ # Use new configurations if there was no preconfiguration
+ if self.config is None:
+ self._config = TaskConfig(**config)
+ # Overwrite configs
+ else:
+ if config is not None:
+ self._config.__dict__.update(config)
+
+ if self.config is None:
+ raise ValueError("Must pass a config to ConfigurableTask, either in cls.CONFIG or `config` kwarg")
+
+ if isinstance(self.config.metadata, dict):
+ if "version" in self.config.metadata:
+ self.VERSION = self.config.metadata["version"]
+
+ self.model_name = model_name
+ self._prepare_model_specific_config()
+
+ if self.config.output_type is not None:
+ if self.config.output_type not in ALL_OUTPUT_TYPES:
+ raise ValueError(f"Got invalid output_type '{self.config.output_type}', must be in '{','.join(ALL_OUTPUT_TYPES)}'")
+ self.OUTPUT_TYPE = self.config.output_type
+
+ if self.config.dataset_path is not None:
+ self.DATASET_PATH = self.config.dataset_path
+
+ if self.config.dataset_name is not None:
+ self.DATASET_NAME = self.config.dataset_name
+
+ self._prepare_metric_and_aggregation()
+
+ self.download(self.config.dataset_kwargs)
+ self._training_docs = None
+ self._fewshot_docs = None
+
+ if self.config.filter_list is not None:
+ self._filters = []
+ for filter_config in self.config.filter_list:
+ for filter_pipeline in filter_config:
+ filter_name = filter_config["name"]
+ filter_functions = filter_config["filter"]
+ components = []
+ for function in filter_functions:
+ kwargs = {key: function[key] for key in function if key != "function"}
+ components.append([function["function"], kwargs])
+ filter_pipeline = build_filter_ensemble(filter_name, components)
+ self._filters.append(filter_pipeline)
+ else:
+ self._filters = [build_filter_ensemble("none", [["take_first", None]])]
+ if self.config.fewshot_config is not None:
+ self.sampler = samplers.get_sampler(self.config.fewshot_config.get("sampler", "default") if self.config.fewshot_config else "default")(list(self.fewshot_docs()), self, rnd=random.Random(1234))
+
+ if self.has_test_docs():
+ self.task_docs = self.test_docs()
+ elif self.has_validation_docs():
+ self.task_docs = self.validation_docs()
+ else:
+ assert False, f"Task dataset (path={self.DATASET_PATH}, name={self.DATASET_NAME}) must have valid or test docs!"
+
+ # Test One Doc
+ self.features = list(self.task_docs.features.keys())
+ self.multiple_input = 0
+ self.multiple_target = 0
+ test_doc = self.task_docs[0]
+ test_text = self.doc_to_text(test_doc)
+ test_target = self.doc_to_target(test_doc)
+
+ if self.config.doc_to_choice is not None:
+ test_choice = self.doc_to_choice(test_doc)
+ if type(test_choice) is not list:
+ eval_logger.error("doc_to_choice must return list")
+ else:
+ num_choice = len(test_choice)
+
+ if type(test_text) is int:
+ self.multiple_input = num_choice
+ else:
+ test_choice = None
+
+ if type(test_target) is list:
+ self.multiple_target = len(test_target)
+ else:
+ if (type(test_target) is int) and (test_choice is not None):
+ test_target = test_choice[test_target]
+ else:
+ test_target = str(test_target)
+
+ if test_choice is not None:
+ check_choices = test_choice
+ else:
+ check_choices = [test_target]
+ if self.config.doc_to_choice is not None:
+ for choice in check_choices:
+ choice_has_whitespace = True if choice[0].isspace() else False
+ delimiter_has_whitespace = True if self.config.target_delimiter.rstrip() != self.config.target_delimiter else False
+
+ if delimiter_has_whitespace and choice_has_whitespace:
+ eval_logger.warning(f'Both target_delimiter and target choice: "{choice}" have whitespace')
+ elif (not delimiter_has_whitespace) and (not choice_has_whitespace):
+ eval_logger.warning(f'Both target_delimiter "{self.config.target_delimiter}" and target choice: "{choice}" do not have whitespace, ignore if the language you are evaluating on does not require/use whitespace')
+
+ def _prepare_model_specific_config(self):
+ self.lmms_eval_specific_kwargs = self.config.lmms_eval_specific_kwargs
+ if self.lmms_eval_specific_kwargs is not None:
+ if self.model_name in self.lmms_eval_specific_kwargs:
+ self.lmms_eval_specific_kwargs = self.lmms_eval_specific_kwargs[self.model_name]
+ elif "default" in self.lmms_eval_specific_kwargs:
+ self.lmms_eval_specific_kwargs.update(self.lmms_eval_specific_kwargs.get("default", {}))
+ elif "dataset" in self.lmms_eval_specific_kwargs:
+ self.lmms_eval_specific_kwargs.update(self.lmms_eval_specific_kwargs.get("dataset", {}))
+
+ self.model_specific_target_kwargs = self.config.model_specific_target_kwargs
+ if self.model_specific_target_kwargs is not None:
+ if self.model_name in self.model_specific_target_kwargs:
+ self.model_specific_target_kwargs = self.model_specific_target_kwargs[self.model_name]
+ else:
+ self.model_specific_target_kwargs = self.model_specific_target_kwargs.get("default", None)
+ self.model_specific_generation_kwargs = self.config.model_specific_generation_kwargs
+ if self.model_specific_generation_kwargs is not None:
+ if self.model_name in self.model_specific_generation_kwargs:
+ self.model_specific_generation_kwargs = self.model_specific_generation_kwargs[self.model_name]
+ else:
+ self.model_specific_generation_kwargs = self.model_specific_generation_kwargs.get("default", {})
+
+ self.config.generation_kwargs.update(self.model_specific_generation_kwargs)
+
+ def _prepare_metric_and_aggregation(self):
+ self._metric_fn_list = {}
+ self._metric_fn_kwargs = {}
+ self._aggregation_list = {}
+ self._higher_is_better = {}
+
+ if self.config.metric_list is None:
+ # TODO: handle this in TaskConfig.__post_init__ ?
+ _metric_list = DEFAULT_METRIC_REGISTRY[self.config.output_type]
+
+ for metric_name in _metric_list:
+ self._metric_fn_list[metric_name] = METRIC_REGISTRY[metric_name]
+ self._metric_fn_kwargs[metric_name] = {}
+ self._aggregation_list[metric_name] = get_metric_aggregation(metric_name)
+ self._higher_is_better[metric_name] = is_higher_better(metric_name)
+ else:
+ for metric_config in self.config.metric_list:
+ assert "metric" in metric_config
+ metric_name = metric_config["metric"]
+ kwargs = {key: metric_config[key] for key in metric_config if key not in ["metric", "aggregation", "higher_is_better"]}
+
+ if self.config.process_results is not None:
+ self._metric_fn_list[metric_name] = None
+ self._metric_fn_kwargs[metric_name] = {}
+ elif callable(metric_name):
+ metric_fn = metric_name.__call__
+ metric_name = metric_name.__name__
+ self._metric_fn_list[metric_name] = metric_fn
+ self._metric_fn_kwargs[metric_name] = kwargs
+ else:
+ self._metric_fn_list[metric_name] = METRIC_REGISTRY[metric_name]
+ self._metric_fn_kwargs[metric_name] = kwargs
+
+ if "aggregation" in metric_config:
+ agg_name = metric_config["aggregation"]
+ if type(agg_name) == str:
+ self._aggregation_list[metric_name] = get_aggregation(agg_name)
+ elif callable(agg_name):
+ self._aggregation_list[metric_name] = metric_config["aggregation"]
+ else:
+ INV_AGG_REGISTRY = {v: k for k, v in AGGREGATION_REGISTRY.items()}
+ metric_agg = get_metric_aggregation(metric_name)
+ eval_logger.warning(f"[Task: {self._config.task}] metric {metric_name} is defined, but aggregation is not. " f"using default " f"aggregation={INV_AGG_REGISTRY[metric_agg]}")
+ self._aggregation_list[metric_name] = metric_agg
+
+ if "higher_is_better" in metric_config:
+ self._higher_is_better[metric_name] = metric_config["higher_is_better"]
+ else:
+ eval_logger.warning(f"[Task: {self._config.task}] metric {metric_name} is defined, but higher_is_better is not. " f"using default " f"higher_is_better={is_higher_better(metric_name)}")
+ self._higher_is_better[metric_name] = is_higher_better(metric_name)
+
+ @retry(stop=(stop_after_attempt(5) | stop_after_delay(60)), wait=wait_fixed(2))
+ def download(self, dataset_kwargs=None) -> None:
+ # If the dataset is a video dataset,
+ # Recursively search whether their is a zip and unzip it to the huggingface home
+ download_config = DownloadConfig()
+ download_config.max_retries = dataset_kwargs.get("max_retries", 10) if dataset_kwargs is not None else 10
+ download_config.num_proc = dataset_kwargs.get("num_proc", 8) if dataset_kwargs is not None else 8
+ download_config.local_files_only = dataset_kwargs.get("local_files_only", False) if dataset_kwargs is not None else False
+ if dataset_kwargs is not None:
+ if "From_YouTube" in dataset_kwargs:
+
+ def _download_from_youtube(path):
+ try:
+ for video in tqdm(self.all_dataset[split]):
+ video_id = video["videoID"]
+ target_path = os.path.join(path, f"{video_id}.mp4")
+ assert shutil.which("yt-dlp") is not None, "yt-dlp must be installed and available in the system's PATH"
+ command = f"yt-dlp -o {target_path} -f mp4 https://www.youtube.com/watch?v={video_id}"
+ subprocess.run(command, shell=True)
+ with open(os.path.join(cache_path, f"{task}_download_status.json"), "w") as f:
+ f.write(json.dumps({task: "downloaded"}))
+ except Exception as e:
+ eval_logger.error(f"Error while downloading {task} data: {e}")
+ with open(os.path.join(cache_path, f"{task}_download_status.json"), "w") as f:
+ f.write(json.dumps({task: "not downloaded"}))
+
+ hf_home = os.getenv("HF_HOME", "~/.cache/huggingface/")
+ accelerator = Accelerator()
+ if accelerator.is_main_process:
+ dataset_kwargs.pop("From_YouTube")
+ assert "load_from_disk" not in dataset_kwargs, "load_from_disk must not be True when From_YouTube is True"
+ self.all_dataset = datasets.load_dataset(
+ path=self.DATASET_PATH,
+ name=self.DATASET_NAME,
+ download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS,
+ **dataset_kwargs if dataset_kwargs is not None else {},
+ )
+ dataset_kwargs["From_YouTube"] = True
+ cache_path = snapshot_download(repo_id=self.DATASET_PATH, repo_type="dataset") # download_parquet
+ split = vars(self.config)["test_split"]
+ task = vars(self.config)["task"]
+
+ video_path = os.path.join(hf_home, task)
+ if os.path.exists(os.path.join(cache_path, f"{task}_download_status.json")):
+ download_status = json.load(open(os.path.join(cache_path, f"{task}_download_status.json"), "r"))
+ if download_status[task] == "downloaded":
+ eval_logger.info(f"Data for {task} already download!")
+ else:
+ eval_logger.info(f"Start downloading YouTube data to {video_path}...")
+ _download_from_youtube(video_path)
+ else:
+ eval_logger.info(f"Start downloading YouTube data to {video_path}...")
+ _download_from_youtube(video_path)
+
+ accelerator.wait_for_everyone()
+ if "builder_script" in dataset_kwargs:
+ builder_script = dataset_kwargs["builder_script"]
+ self.DATASET_PATH = os.path.join(cache_path, builder_script)
+ dataset_kwargs.pop("builder_script")
+
+ downloaded_video_ids = [i.split(".mp4")[0] for i in os.listdir(os.path.expanduser(video_path)) if i.endswith(".mp4")]
+ # Filtered the existing dataset with the downloaded video ids
+ self.dataset = datasets.DatasetDict({split: self.all_dataset[split].filter(lambda x: x["videoID"] in downloaded_video_ids)})
+
+ self.dataset_no_image = self.dataset
+ dataset_kwargs.pop("From_YouTube")
+ return
+
+ if "video" in dataset_kwargs and dataset_kwargs["video"]:
+ hf_home = os.getenv("HF_HOME", "~/.cache/huggingface/")
+ hf_home = os.path.expanduser(hf_home)
+ cache_dir = dataset_kwargs["cache_dir"]
+ cache_dir = os.path.join(hf_home, cache_dir)
+ accelerator = Accelerator()
+ if accelerator.is_main_process:
+ force_download = dataset_kwargs.get("force_download", False)
+ force_unzip = dataset_kwargs.get("force_unzip", False)
+ revision = dataset_kwargs.get("revision", "main")
+ create_link = dataset_kwargs.get("create_link", False)
+ # If the user already has a cache dir, we skip download the zip files
+ if not os.path.exists(cache_dir):
+ cache_path = snapshot_download(repo_id=self.DATASET_PATH, revision=revision, repo_type="dataset", force_download=force_download, etag_timeout=60)
+ zip_files = glob(os.path.join(cache_path, "**/*.zip"), recursive=True)
+ tar_files = glob(os.path.join(cache_path, "**/*.tar*"), recursive=True)
+ else:
+ zip_files = []
+ tar_files = []
+
+ def unzip_video_data(zip_file):
+ import os
+ import zipfile
+
+ with zipfile.ZipFile(zip_file, "r") as zip_ref:
+ for file_info in zip_ref.infolist():
+ target_path = os.path.join(cache_dir, file_info.filename)
+ if not os.path.exists(target_path):
+ zip_ref.extract(file_info, cache_dir)
+ else:
+ eval_logger.info(f"Skipping existing file: {target_path}")
+
+ eval_logger.info(f"Extracted all files from {zip_file} to {cache_dir}")
+
+ def untar_video_data(tar_file):
+ import tarfile
+
+ with tarfile.open(tar_file, "r") as tar_ref:
+ tar_ref.extractall(cache_dir)
+ eval_logger.info(f"Extracted all files from {tar_file} to {cache_dir}")
+
+ def concat_tar_parts(tar_parts, output_tar):
+ with open(output_tar, "wb") as out_tar:
+ from tqdm import tqdm
+
+ for part in tqdm(sorted(tar_parts)):
+ with open(part, "rb") as part_file:
+ out_tar.write(part_file.read())
+ eval_logger.info(f"Concatenated parts {tar_parts} into {output_tar}")
+
+ # Unzip zip files if needed
+ if force_unzip or (not os.path.exists(cache_dir) and len(zip_files) > 0):
+ for zip_file in zip_files:
+ unzip_video_data(zip_file)
+
+ # Concatenate and extract tar files if needed
+ if force_unzip or (not os.path.exists(cache_dir) and len(tar_files) > 0):
+ tar_parts_dict = {}
+
+ # Group tar parts together
+ for tar_file in tar_files:
+ base_name = tar_file.split(".tar")[0]
+ base_name = re.sub(r"_\d+$", "", base_name)
+ if base_name not in tar_parts_dict:
+ tar_parts_dict[base_name] = []
+ tar_parts_dict[base_name].append(tar_file)
+
+ # Concatenate and untar split parts
+ for base_name, parts in tar_parts_dict.items():
+ eval_logger.info(f"Extracting following tar files: {parts}")
+ output_tar = base_name + ".tar"
+ if not os.path.exists(output_tar):
+ eval_logger.info(f"Start concatenating tar files")
+
+ concat_tar_parts(parts, output_tar)
+ eval_logger.info(f"Finish concatenating tar files")
+
+ if not os.path.exists(os.path.join(cache_dir, os.path.basename(base_name))):
+ untar_video_data(output_tar)
+
+ # Link cache_path to cache_dir if needed.
+ if create_link:
+ if not os.path.exists(cache_dir) or os.path.islink(cache_dir):
+ if os.path.islink(cache_dir):
+ os.remove(cache_dir)
+ eval_logger.info(f"Removed existing symbolic link: {cache_dir}")
+ # Create a new symbolic link
+ os.symlink(cache_path, cache_dir)
+ eval_logger.info(f"Symbolic link created successfully: {cache_path} -> {cache_dir}")
+
+ accelerator.wait_for_everyone()
+ dataset_kwargs.pop("cache_dir")
+ dataset_kwargs.pop("video")
+
+ if "builder_script" in dataset_kwargs:
+ builder_script = dataset_kwargs["builder_script"]
+ self.DATASET_PATH = os.path.join(cache_path, builder_script)
+ dataset_kwargs.pop("builder_script")
+
+ if "force_download" in dataset_kwargs:
+ dataset_kwargs.pop("force_download")
+
+ if "force_unzip" in dataset_kwargs:
+ dataset_kwargs.pop("force_unzip")
+
+ if "local_files_only" in dataset_kwargs:
+ dataset_kwargs.pop("local_files_only")
+
+ if "create_link" in dataset_kwargs:
+ dataset_kwargs.pop("create_link")
+
+ if dataset_kwargs is not None and "load_from_disk" in dataset_kwargs and dataset_kwargs["load_from_disk"]:
+ # using local task in offline environment, need to process the online dataset into local format via
+ # `ds = load_datasets("lmms-lab/MMMU")`
+ self.dataset = datasets.load_from_disk(dataset_path=self.DATASET_PATH)
+ else:
+ self.dataset = datasets.load_dataset(
+ path=self.DATASET_PATH,
+ name=self.DATASET_NAME,
+ download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS,
+ download_config=download_config,
+ **dataset_kwargs if dataset_kwargs is not None else {},
+ )
+
+ if self.config.process_docs is not None:
+ for split in self.dataset:
+ if split in [self.config.training_split, self.config.validation_split, self.config.test_split, self.config.fewshot_split]:
+ self.dataset[split] = self.config.process_docs(self.dataset[split])
+
+ # copy dataset, remove image features
+ self.dataset_no_image = self.dataset.copy()
+ for doc_name in self.dataset_no_image:
+ remove_cols = []
+ features = self.dataset_no_image[doc_name].features
+ # If it is an Image instance or a Sequence of Image instance. Remove it
+ for feature in features:
+ if isinstance(features[feature], Image):
+ remove_cols.append(feature)
+ elif isinstance(features[feature], Sequence) and isinstance(features[feature].feature, Image):
+ remove_cols.append(feature)
+ elif isinstance(features[feature], Audio):
+ remove_cols.append(feature)
+ for remove_col in remove_cols:
+ self.dataset_no_image[doc_name] = self.dataset_no_image[doc_name].remove_columns(remove_col)
+
+ def has_training_docs(self) -> bool:
+ if self.config.training_split is not None:
+ return True
+ else:
+ return False
+
+ def has_validation_docs(self) -> bool:
+ if self.config.validation_split is not None:
+ return True
+ else:
+ return False
+
+ def has_test_docs(self) -> bool:
+ if self.config.test_split is not None:
+ return True
+ else:
+ return False
+
+ def training_docs(self) -> datasets.Dataset:
+ if self.has_training_docs():
+ return self.dataset[self.config.training_split]
+
+ def validation_docs(self) -> datasets.Dataset:
+ if self.has_validation_docs():
+ return self.dataset[self.config.validation_split]
+
+ def validation_docs_no_media(self) -> datasets.Dataset:
+ if self.has_validation_docs():
+ return self.dataset_no_image[self.config.validation_split]
+
+ def test_docs(self) -> datasets.Dataset:
+ if self.has_test_docs():
+ return self.dataset[self.config.test_split]
+
+ def test_docs_no_media(self) -> datasets.Dataset:
+ if self.has_test_docs():
+ return self.dataset_no_image[self.config.test_split]
+
+ @property
+ def eval_docs_no_media(self) -> Union[datasets.Dataset, List[dict]]:
+ if self.has_test_docs():
+ return self.test_docs_no_media()
+ elif self.has_validation_docs():
+ return self.validation_docs_no_media()
+ else:
+ raise ValueError(f"Task dataset (path={self.DATASET_PATH}, name={self.DATASET_NAME}) must have valid or test docs!")
+
+ def fewshot_docs(self):
+ if self.config.fewshot_split is not None:
+ return self.dataset[self.config.fewshot_split]
+ else:
+ if (self.config.num_fewshot is not None) and (self.config.num_fewshot > 0):
+ eval_logger.warning(f"Task '{self.config.task}': " "num_fewshot > 0 but fewshot_split is None. " "using preconfigured rule.")
+ return super().fewshot_docs()
+
+ @utils.positional_deprecated
+ def fewshot_context(
+ self,
+ doc: str,
+ num_fewshot: int,
+ system_instruction: Optional[str] = None,
+ apply_chat_template: bool = False,
+ fewshot_as_multiturn: bool = False,
+ chat_template: Optional[Callable] = None,
+ is_multimodal: bool = False,
+ ) -> str:
+ """Returns a fewshot context string that is made up of a prepended description
+ (if provided), the `num_fewshot` number of examples, and an appended prompt example.
+
+ :param doc: str
+ The document as returned from training_docs, validation_docs, or test_docs.
+ :param num_fewshot: int
+ The number of fewshot examples to provide in the returned context string.
+ :param system_instruction: str
+ System instruction to be applied to the prompt.
+ :param apply_chat_template: bool
+ Whether to apply the chat template to the fewshot context.
+ :param fewshot_as_multiturn: bool
+ Whether to provide the fewshot examples as a multiturn conversation or a single user turn.
+ :param chat_template:
+ callable (from lm.apply_chat_template) that takes in a list[Dict] chat transcript and renders it into a string.
+ :returns: str
+ The fewshot context.
+ """
+
+ if apply_chat_template:
+ labeled_examples = []
+ else:
+ labeled_examples = ""
+
+ # get task description
+ if description := self.config.description:
+ description = utils.apply_template(self.config.description, doc)
+
+ # create system prompt based on the provided system instruction and description
+ if system_instruction is not None and description:
+ system_prompt = f"{system_instruction}{self.sampler.fewshot_delimiter}{description}"
+ elif system_instruction is not None:
+ system_prompt = system_instruction
+ elif description:
+ system_prompt = description
+ else:
+ system_prompt = ""
+
+ # add system prompt if specified
+ if system_prompt:
+ if apply_chat_template:
+ labeled_examples.append({"role": "system", "content": system_prompt})
+ else:
+ labeled_examples = system_prompt
+
+ # if few-shot - append examples after the system prompt
+ if num_fewshot > 0:
+ if is_multimodal is False:
+ if apply_chat_template:
+ labeled_examples.extend(self.sampler.get_chat_context(doc, num_fewshot, fewshot_as_multiturn))
+ else:
+ labeled_examples += self.sampler.get_context(doc, num_fewshot)
+ else:
+ if apply_chat_template:
+ labeled_examples_text, labeled_examples_multimodal = self.sampler.get_multimodal_chat_context(doc, num_fewshot, fewshot_as_multiturn)
+ labeled_examples.extend(labeled_examples_text)
+ else:
+ labeled_examples_text, labeled_examples_multimodal = self.sampler.get_multimodal_context(doc, num_fewshot)
+ labeled_examples += labeled_examples_text
+
+ example = self.doc_to_text(doc)
+ if is_multimodal is False:
+ if apply_chat_template:
+ if self.multiple_input:
+ return chat_template(labeled_examples)
+ if isinstance(example, str):
+ self.append_target_question(labeled_examples, example, fewshot_as_multiturn)
+ # for loglikelihood create a list of questions with appended choices
+ elif isinstance(example, list):
+ labeled_examples_list = []
+ # copy chat history for each example and append the answer
+ for ex in example:
+ chat = copy.deepcopy(labeled_examples)
+ self.append_target_question(chat, ex, fewshot_as_multiturn)
+ labeled_examples_list.append(chat_template(chat))
+ return labeled_examples_list
+ # if example is an integer, append the choice or convert to string
+ elif isinstance(example, int):
+ if self.config.doc_to_choice is not None:
+ choices = self.doc_to_choice(doc)
+ self.append_target_question(labeled_examples, choices[example], fewshot_as_multiturn)
+ else:
+ self.append_target_question(labeled_examples, str(example), fewshot_as_multiturn)
+ # return lm.apply_chat_template(labeled_examples)
+ return chat_template(labeled_examples)
+ else:
+ if self.multiple_input:
+ return labeled_examples
+ if isinstance(example, str):
+ return labeled_examples + example
+ elif isinstance(example, list):
+ return [labeled_examples + ex for ex in example]
+ elif isinstance(example, int):
+ if self.config.doc_to_choice is not None:
+ choices = self.doc_to_choice(doc)
+ return labeled_examples + choices[example]
+ else:
+ return labeled_examples + str(example)
+ else:
+ if apply_chat_template:
+ raise NotImplementedError("Multimodal chat template not implemented yet")
+ else:
+ if self.multiple_input:
+ return labeled_examples + " " + example, labeled_examples_multimodal
+ if isinstance(example, str):
+ return labeled_examples + " " + example, labeled_examples_multimodal
+ else:
+ raise NotImplementedError("Multimodal not implemented yet")
+ # elif isinstance(example, list):
+ # return [labeled_examples + ex for ex in example]
+ # elif isinstance(example, int):
+ # if self.config.doc_to_choice is not None:
+ # choices = self.doc_to_choice(doc)
+ # return labeled_examples + choices[example], labeled_examples_multimodal
+ # else:
+ # return labeled_examples + str(example), labeled_examples_multimodal
+
+ def apply_filters(self) -> Optional[List[Instance]]:
+ """Iterates over FilterEnsembles and applies them to instances"""
+ if hasattr(self, "_filters"):
+ for f in self._filters:
+ f.apply(self._instances, self.task_docs)
+ else:
+ eval_logger.warning("No filter defined, passing through instances")
+ return self._instances
+
+ def should_decontaminate(self):
+ return self.config.should_decontaminate
+
+ def doc_to_decontamination_query(self, doc):
+ if self.config.should_decontaminate:
+ if self.config.doc_to_decontamination_query is None:
+ return self.doc_to_text(doc)
+ else:
+ doc_to_decontamination_query = self.config.doc_to_decontamination_query
+ if doc_to_decontamination_query in self.features:
+ return doc[doc_to_decontamination_query]
+ elif callable(doc_to_decontamination_query):
+ return doc_to_decontamination_query(doc)
+ else:
+ return ast.literal_eval(utils.apply_template(self.config.doc_to_decontamination_query, doc))
+
+ def _process_doc(self, doc):
+ """
+ Override this to process (detokenize, strip, replace, etc.) individual
+ documents. This can be used in a map over documents of a data split.
+ E.g. `map(self._process_doc, self.dataset["validation"])`
+
+ :return: dict
+ The processed version of the specified `doc`.
+ """
+ return doc
+
+ def doc_to_text(self, doc):
+ doc_to_text = self.config.doc_to_text
+
+ if type(doc_to_text) == int:
+ return doc_to_text
+ elif type(doc_to_text) == str:
+ if doc_to_text in self.features:
+ # if self.config.doc_to_choice is not None:
+ # return self.doc_to_choice(doc)[doc[doc_to_text]]
+ # else:
+ return doc[doc_to_text]
+ else:
+ text_string = utils.apply_template(doc_to_text, doc)
+ if text_string.isdigit() and self._config.doc_to_choice is not None:
+ return ast.literal_eval(text_string)
+ else:
+ return text_string
+ elif callable(doc_to_text):
+ return (
+ doc_to_text(doc, self.lmms_eval_specific_kwargs)
+ if self.lmms_eval_specific_kwargs is not None
+ else doc_to_text(
+ doc,
+ )
+ )
+ # Used when applying a Promptsource template
+ elif hasattr(doc_to_text, "apply"):
+ applied_prompt = doc_to_text.apply(doc)
+ if len(applied_prompt) == 2:
+ return applied_prompt[0]
+ else:
+ eval_logger.warning("Applied prompt returns empty string")
+ return self.config.fewshot_delimiter
+ else:
+ print(type(doc_to_text))
+ raise TypeError
+
+ def doc_to_target(self, doc: dict) -> Union[int, str, list]:
+ doc_to_target = self.config.doc_to_target
+
+ if type(doc_to_target) == int:
+ return doc_to_target
+ elif type(doc_to_target) == str:
+ if doc_to_target in self.features:
+ # if self.config.doc_to_choice is not None:
+ # return self.doc_to_choice(doc)[doc[doc_to_target]]
+ # else:
+ return doc[doc_to_target]
+ else:
+ target_string = utils.apply_template(doc_to_target, doc)
+ if target_string.isdigit() and self._config.doc_to_choice is not None:
+ return ast.literal_eval(target_string)
+ elif len(target_string) >= 2 and (target_string[0] == "[") and (target_string[-1] == "]"):
+ try:
+ return ast.literal_eval(target_string)
+ except (SyntaxError, ValueError):
+ return target_string
+ else:
+ return target_string
+ elif type(doc_to_target) == list:
+ return doc_to_target
+ elif callable(doc_to_target):
+ return doc_to_target(doc, self.model_specific_target_kwargs) if self.model_specific_target_kwargs is not None else doc_to_target(doc)
+ # Used when applying a Promptsource template
+ elif hasattr(doc_to_target, "apply"):
+ applied_prompt = doc_to_target.apply(doc)
+ if len(applied_prompt) == 2:
+ return applied_prompt[1]
+ else:
+ eval_logger.warning("Applied prompt returns empty string")
+ return self.config.fewshot_delimiter
+ else:
+ raise TypeError
+
+ def doc_to_visual(self, doc: dict) -> Union[int, str, list]:
+ self.config.doc_to_visual
+ if type(self.config.doc_to_visual) == str:
+ assert self.config.doc_to_visual in self.features
+ # Single image. Still return a list for consistency.
+ return [doc[self.config.doc_to_visual]]
+ elif callable(self.config.doc_to_visual):
+ return (
+ self.config.doc_to_visual(doc, self.lmms_eval_specific_kwargs)
+ if self.lmms_eval_specific_kwargs is not None and len(inspect.signature(self.config.doc_to_visual).parameters) == 2
+ else self.config.doc_to_visual(
+ doc,
+ )
+ )
+ else:
+ # eval_logger.warning("Note that doc_to_visual was called but not set in config. Please check if this is a text-only task.")
+ return self.config.doc_to_visual
+
+ def doc_to_choice(self, doc: Any) -> List[str]:
+ if self.config.doc_to_choice is None:
+ eval_logger.error("Note that doc_to_choice was called but not set in config.")
+ else:
+ doc_to_choice = self.config.doc_to_choice
+
+ if type(doc_to_choice) == str:
+ if doc_to_choice in self.features:
+ return doc[doc_to_choice]
+ else:
+ return ast.literal_eval(utils.apply_template(doc_to_choice, doc))
+ elif type(doc_to_choice) == list:
+ return doc_to_choice
+ elif type(doc_to_choice) == dict:
+ return list(doc_to_choice.values())
+ elif callable(doc_to_choice):
+ return doc_to_choice(doc)
+ elif hasattr(doc_to_choice, "get_answer_choices_list"):
+ return doc_to_choice.get_answer_choices_list(doc)
+ else:
+ raise TypeError
+
+ def construct_requests(self, doc_id: int, ctx: str, **kwargs) -> Union[List[Instance], Instance]:
+ split = kwargs.get("metadata").get("split")
+ # kwargs.pop("split")
+ if self.OUTPUT_TYPE == "loglikelihood":
+ arguments = (ctx, self.doc_to_target, self.doc_to_visual, doc_id, self.config.task, split)
+ elif self.OUTPUT_TYPE == "multiple_choice":
+ doc = self.dataset[split][doc_id]
+ choices = self.doc_to_choice(doc)
+ target_delimiter = self.config.target_delimiter
+ if self.multiple_input:
+ # If there are multiple inputs, choices are placed in the ctx
+ cont = self.doc_to_target(doc)
+ arguments = [(ctx, f"{target_delimiter}{cont}", self.doc_to_visual, doc_id, self.config.task, split) for ctx in choices]
+ else:
+ # Otherwise they are placed in the continuation
+ arguments = [(ctx, f"{target_delimiter}{cont}", self.doc_to_visual, doc_id, self.config.task, split) for cont in choices]
+ request_list = [
+ Instance(
+ request_type="loglikelihood",
+ # doc=doc,
+ arguments=arg,
+ idx=i,
+ **kwargs,
+ )
+ for i, arg in enumerate(arguments)
+ ]
+ # TODO: we should raise a warning telling users this will at most ~2x runtime.
+ if "acc_mutual_info" in self._metric_fn_list.keys():
+ # if we are calculating multiple choice accuracy
+ # using mutual information instead of raw loglikelihood as metric, need unconditional lls.
+
+ # here mutual info refers to calculating
+ # log(P(choice|ctx) / P(choice)) = log(P(choice|ctx)) - log(P(choice))
+ # in other words normalizing by subtracting the unconditional logprob of each choice.
+ request_list.extend(
+ [
+ Instance(
+ request_type="loglikelihood",
+ # doc=doc,
+ arguments=("", "{}".format(choice)),
+ idx=i,
+ **kwargs,
+ )
+ for i, choice in enumerate(choices)
+ ]
+ )
+ return request_list
+
+ elif self.OUTPUT_TYPE == "generate_until":
+ arguments = (ctx, copy.deepcopy(self.config.generation_kwargs), self.doc_to_visual, doc_id, self.config.task, split)
+ elif self.OUTPUT_TYPE == "generate_until_multi_round":
+ arguments = (ctx, copy.deepcopy(self.config.generation_kwargs), self.doc_to_visual, partial(self.config.doc_to_text, lmms_eval_specific_kwargs=self.lmms_eval_specific_kwargs), doc_id, self.config.task, split)
+ return Instance(request_type=self.OUTPUT_TYPE, arguments=arguments, idx=0, **kwargs)
+
+ # TODO: we add a full_docs interface here for some evaluations that needs to access the full datasets during process_results function. we may have better ways to handle this.
+ @retry(stop=(stop_after_attempt(5) | stop_after_delay(1200)), wait=wait_fixed(2))
+ def process_results(self, doc, results, full_docs=None):
+ if self.OUTPUT_TYPE == "generate_until":
+ if isinstance(results, list) and isinstance(results[0], list):
+ results = [res.strip() for res in results[0]]
+ else:
+ results = [res.strip() for res in results]
+
+ kwargs = {}
+ if full_docs is not None:
+ kwargs["full_docs"] = full_docs
+ if callable(self.config.process_results):
+ return self.config.process_results(doc, results, **kwargs)
+
+ result_dict = {}
+ use_metric = list(self._metric_fn_list.keys())
+ if self.OUTPUT_TYPE == "loglikelihood":
+ ll, is_greedy = results
+ return {
+ **({"perplexity": ll} if "perplexity" in use_metric else {}),
+ **({"acc": int(is_greedy)} if "acc" in use_metric else {}),
+ }
+ elif self.OUTPUT_TYPE == "multiple_choice":
+ lls, is_greedy = zip(*results)
+
+ # retrieve choices in List[str] form, to compute choice lengths, etc.
+ choices = self.doc_to_choice(doc)
+ completion_len = np.array([float(len(i)) for i in choices])
+
+ if 2 * len(choices) == len(lls) and "acc_mutual_info" in self._metric_fn_list.keys():
+ # then we are doing mutual info.
+ # this stores the "dryrun" / unconditional answer loglikelihoods
+ lls_unconditional = lls[1::2]
+ assert len(lls_unconditional) == len(choices)
+ # and this stores our "regular" conditional loglikelihoods
+ lls = lls[::2]
+
+ # Warning :
+ # Here may be different from original lm-eval
+ # since we return the actual loss in many model loglikelihood
+ # we just use the argmin here
+ pred = np.argmin(lls)
+ pred_norm = np.argmin(lls / completion_len)
+
+ if self.multiple_input:
+ gold = self.doc_to_text(doc)
+ else:
+ gold = self.doc_to_target(doc)
+
+ gold_index_error = False
+ if type(gold) is list:
+ gold = [i if i < len(choices) else -100 for i in gold]
+ if -100 in gold:
+ gold_index_error = True
+ else:
+ if type(gold) is int:
+ gold = gold if gold < len(choices) else -100
+ elif type(gold) is str:
+ gold = choices.index(gold) if gold in choices else -100
+
+ if gold == -100:
+ gold_index_error = True
+
+ if gold_index_error:
+ eval_logger.warning(f"Label index was not in within range of available choices," f"Sample:\n\n{doc}\n\n")
+
+ if self.multiple_target:
+ acc = 1.0 if pred in gold else 0.0
+ acc_norm = 1.0 if pred_norm in gold else 0.0
+ exact_match = int(any([is_greedy[i] if i != -100 else 0 for i in gold]))
+ else:
+ acc = 1.0 if pred == gold else 0.0
+ acc_norm = 1.0 if pred_norm == gold else 0.0
+ # TODO: this gets score of 0 on arc_challenge for pythia-70m. need to test that this works properly
+ exact_match = int(is_greedy[gold]) if gold != -100 else 0
+
+ result_dict = {
+ **({"acc": acc} if "acc" in use_metric else {}),
+ **({"f1": (gold, pred)} if "f1" in use_metric else {}),
+ **({"mcc": (gold, pred)} if "mcc" in use_metric else {}),
+ **({"acc_norm": acc_norm} if "acc_norm" in use_metric else {}),
+ **({"exact_match": exact_match} if "exact_match" in use_metric else {}),
+ }
+
+ if "acc_mutual_info" in use_metric:
+ lls_mutual_info = [ll_c - ll_u for ll_c, ll_u in zip(lls, lls_unconditional)]
+ acc_mutual_info = 1.0 if np.argmax(lls_mutual_info) == gold else 0.0
+ result_dict["acc_mutual_info"] = acc_mutual_info
+
+ elif "generate_until" in self.OUTPUT_TYPE:
+ gold = self.doc_to_target(doc)
+ result = [res.strip() for res in results]
+ if self.config.doc_to_choice is not None:
+ # If you set doc_to_choice,
+ # it assumes that doc_to_target returns a number.
+ choices = self.doc_to_choice(doc)
+ gold = choices[gold]
+ # we expect multiple_targets to be a list.
+ elif self.multiple_target:
+ gold = list(gold)
+ # elif type(gold) != type(result):
+ # # cast gold to the same type as result
+ # gold = type(result)(gold)
+
+ for metric in self._metric_fn_list.keys():
+ if self.multiple_target and metric != "anls":
+ # in the case where we have multiple targets,
+ # return true if any are true
+ # TODO: this may break for multipLe_target, non zero-or-1 metrics
+ scores = []
+ if not isinstance(gold, list):
+ # sometimes, a multiple_target dataset has exceptions where one doc has only one string answer
+ # print(gold)
+ gold = [gold]
+ for gold_option in gold:
+ try:
+ result_score = self._metric_fn_list[metric](
+ references=[gold_option],
+ predictions=result,
+ **self._metric_fn_kwargs[metric],
+ )
+ except TypeError: # TODO: this is hacky and I don't want to do it
+ result_score = self._metric_fn_list[metric]([gold_option, result])
+ if isinstance(result_score, dict):
+ # TODO: this handles the case where HF evaluate returns a dict.
+ result_score = result_score[metric]
+ scores.append(result_score)
+ if any(scores):
+ result_score = 1.0
+ else:
+ result_score = 0.0
+ else:
+ if not isinstance(gold, list):
+ gold = [gold]
+ try:
+ result_score = self._metric_fn_list[metric](
+ references=gold,
+ predictions=result,
+ **self._metric_fn_kwargs[metric],
+ )
+ except TypeError: # needed for now in order to use a different interface between our own metrics and HF Evaluate metrics
+ result_score = self._metric_fn_list[metric]([gold, result])
+ if isinstance(result_score, dict):
+ # TODO: this handles the case where HF evaluate returns a dict.
+ result_score = result_score[metric]
+ result_dict[metric] = result_score
+ else:
+ raise ValueError(
+ f"Passed invalid output_type '{self.OUTPUT_TYPE}' ! Please use one of ",
+ "'loglikelihood','generate_until', 'generate_until_multi_round', or 'multiple_choice'",
+ )
+
+ return result_dict
+
+ def aggregation(self):
+ return self._aggregation_list
+
+ def higher_is_better(self):
+ return self._higher_is_better
+
+ def get_config(self, key: str) -> Any:
+ return getattr(self._config, key, None)
+
+ @property
+ def task_name(self) -> Any:
+ return getattr(self.config, "task", None)
+
+ def __repr__(self):
+ return f"ConfigurableTask(task_name={getattr(self.config, 'task', None)}," f"output_type={self.OUTPUT_TYPE}," f"num_fewshot={getattr(self.config, 'num_fewshot', None)}," f"num_samples={len(self.eval_docs)})"
+
+
+class ConfigurableMessagesTask(ConfigurableTask):
+ def __init__(self, data_dir=None, cache_dir=None, download_mode=None, config=None, model_name=None):
+ super().__init__(data_dir, cache_dir, download_mode, config, model_name)
+
+ def doc_to_messages(self, doc: dict) -> Union[int, str, list]:
+ if callable(self.config.doc_to_messages):
+ return (
+ self.config.doc_to_messages(doc, self.lmms_eval_specific_kwargs)
+ if self.lmms_eval_specific_kwargs is not None and len(inspect.signature(self.config.doc_to_messages).parameters) == 2
+ else self.config.doc_to_messages(
+ doc,
+ )
+ )
+ elif self.config.doc_to_messages is None and (self.config.doc_to_visual is not None or self.config.doc_to_text is not None):
+ # An auto doc to messages function
+ def auto_doc_to_messages(doc):
+ visuals = self.doc_to_visual(doc)
+ if visuals is None:
+ visuals = []
+ text = self.doc_to_text(doc)
+ messages = [{"role": "user", "content": []}]
+ content = []
+ for visual in visuals:
+ if isinstance(visual, PIL_Image.Image):
+ content.append({"type": "image", "url": visual})
+ elif isinstance(visual, dict):
+ content.append({"type": "audio", "url": visual})
+ elif isinstance(visual, str):
+ content.append({"type": "video", "url": visual})
+ content.append({"type": "text", "text": text})
+ messages[0]["content"] = content
+ return messages
+
+ return auto_doc_to_messages(doc)
+ else:
+ # eval_logger.warning("Note that doc_to_visual was called but not set in config. Please check if this is a text-only task.")
+ return self.config.doc_to_messages
+
+ def construct_requests(self, doc_id: int, ctx: str, **kwargs) -> Union[List[Instance], Instance]:
+ split = kwargs.get("metadata").get("split")
+ # kwargs.pop("split")
+ assert self.OUTPUT_TYPE == "generate_until", "Currently messages is used for generation only"
+
+ arguments = (ctx, self.doc_to_messages, copy.deepcopy(self.config.generation_kwargs), doc_id, self.config.task, split)
+ return Instance(request_type=self.OUTPUT_TYPE, arguments=arguments, idx=0, **kwargs)
+
+ def __repr__(self):
+ return f"ConfigurableMessagesTask(task_name={getattr(self.config, 'task', None)}," f"output_type={self.OUTPUT_TYPE}," f"num_fewshot={getattr(self.config, 'num_fewshot', None)}," f"num_samples={len(self.eval_docs)})"
diff --git a/lmms-eval/lmms_eval/caching/__init__.py b/lmms-eval/lmms_eval/caching/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lmms-eval/lmms_eval/caching/cache.py b/lmms-eval/lmms_eval/caching/cache.py
new file mode 100644
index 0000000000000000000000000000000000000000..511ecb8f5c43628a270516b755766979185c338e
--- /dev/null
+++ b/lmms-eval/lmms_eval/caching/cache.py
@@ -0,0 +1,68 @@
+import hashlib
+import os
+import pickle
+
+import dill
+
+from lmms_eval.loggers.utils import _handle_non_serializable, is_serializable
+from lmms_eval.utils import eval_logger
+
+MODULE_DIR = os.path.dirname(os.path.realpath(__file__))
+
+OVERRIDE_PATH = os.getenv("LM_HARNESS_CACHE_PATH")
+
+
+PATH = OVERRIDE_PATH if OVERRIDE_PATH else f"{MODULE_DIR}/.cache"
+
+# This should be sufficient for uniqueness
+HASH_INPUT = "EleutherAI-lm-evaluation-harness"
+
+HASH_PREFIX = hashlib.sha256(HASH_INPUT.encode("utf-8")).hexdigest()
+
+FILE_SUFFIX = f".{HASH_PREFIX}.pickle"
+
+
+def load_from_cache(file_name):
+ try:
+ path = f"{PATH}/{file_name}{FILE_SUFFIX}"
+
+ with open(path, "rb") as file:
+ cached_task_dict = dill.loads(file.read())
+ return cached_task_dict
+
+ except Exception:
+ eval_logger.debug(f"{file_name} is not cached, generating...")
+ pass
+
+
+def save_to_cache(file_name, obj):
+ if not os.path.exists(PATH):
+ os.mkdir(PATH)
+
+ file_path = f"{PATH}/{file_name}{FILE_SUFFIX}"
+
+ serializable_obj = []
+
+ for item in obj:
+ for subitem in item:
+ if hasattr(subitem, "arguments"): # we need to handle the arguments specially since doc_to_visual is callable method and not serializable
+ serializable_arguments = tuple(arg if not callable(arg) else None for arg in subitem.arguments)
+ subitem.arguments = serializable_arguments
+
+ eval_logger.debug(f"Saving {file_path} to cache...")
+ try:
+ with open(file_path, "wb") as file:
+ file.write(dill.dumps(serializable_obj))
+ except (pickle.PickleError, dill.PicklingError, TypeError, AttributeError):
+ with open(file_path, "wb") as file:
+ file.write(dill.dumps([[subitem if is_serializable(subitem) else _handle_non_serializable(subitem) for subitem in item] for item in obj]))
+
+
+# NOTE the "key" param is to allow for flexibility
+def delete_cache(key: str = ""):
+ files = os.listdir(PATH)
+
+ for file in files:
+ if file.startswith(key) and file.endswith(FILE_SUFFIX):
+ file_path = f"{PATH}/{file}"
+ os.unlink(file_path)
diff --git a/lmms-eval/lmms_eval/evaluator.py b/lmms-eval/lmms_eval/evaluator.py
new file mode 100644
index 0000000000000000000000000000000000000000..6939bae7a743bd7c9c92bc6420e20e72c04a5fd7
--- /dev/null
+++ b/lmms-eval/lmms_eval/evaluator.py
@@ -0,0 +1,717 @@
+import collections
+import inspect
+import itertools
+import json
+import os
+import random
+import sys
+import time
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Callable, List, Optional, Union
+
+import numpy as np
+import torch
+import torch.distributed as dist
+from accelerate import Accelerator
+from datasets import Image, Sequence
+from loguru import logger as eval_logger
+from tqdm import tqdm
+
+import lmms_eval.api
+import lmms_eval.api.metrics
+import lmms_eval.api.registry
+from lmms_eval.evaluator_utils import (
+ consolidate_group_results,
+ consolidate_results,
+ get_sample_size,
+ get_subtask_list,
+ get_task_list,
+ prepare_print_tasks,
+ print_writeout,
+ run_task_tests,
+)
+from lmms_eval.llm_judge.launcher import get_launcher
+from lmms_eval.loggers.evaluation_tracker import EvaluationTracker
+from lmms_eval.models import get_model
+from lmms_eval.tasks import TaskManager, get_task_dict
+from lmms_eval.utils import (
+ create_iterator,
+ get_datetime_str,
+ get_git_commit_hash,
+ handle_non_serializable,
+ hash_string,
+ make_table,
+ positional_deprecated,
+ run_task_tests,
+ simple_parse_args_string,
+)
+
+
+@positional_deprecated
+def simple_evaluate(
+ model,
+ model_args: Optional[Union[str, dict]] = None,
+ launcher_args: Optional[Union[str, dict]] = None,
+ tasks: Optional[List[Union[str, dict, object]]] = None,
+ num_fewshot: Optional[int] = None,
+ batch_size: Optional[Union[int, str]] = None,
+ max_batch_size: Optional[int] = None,
+ device: Optional[str] = None,
+ use_cache: Optional[str] = None,
+ cache_requests: bool = False,
+ rewrite_requests_cache: bool = False,
+ delete_requests_cache: bool = False,
+ limit: Optional[Union[int, float]] = None,
+ bootstrap_iters: int = 100000,
+ check_integrity: bool = False,
+ write_out: bool = False,
+ log_samples: bool = True,
+ evaluation_tracker: Optional[EvaluationTracker] = None,
+ system_instruction: Optional[str] = None,
+ apply_chat_template: bool = False,
+ fewshot_as_multiturn: bool = False,
+ gen_kwargs: Optional[str] = None,
+ task_manager: Optional[TaskManager] = None,
+ verbosity: str = "INFO",
+ predict_only: bool = False,
+ random_seed: int = 0,
+ numpy_random_seed: int = 1234,
+ torch_random_seed: int = 1234,
+ fewshot_random_seed: int = 1234,
+ datetime_str: str = get_datetime_str(),
+ distributed_executor_backend: str = "accelerate",
+ cli_args=None,
+ force_simple: bool = False,
+):
+ """Instantiate and evaluate a model on a list of tasks.
+
+ :param model: Union[str, LM]
+ Name of model or LM object, see lmms_eval.models.get_model
+ :param model_args: Optional[str, dict]
+ String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
+ Ignored if `model` argument is a LM object.
+ :param tasks: list[Union[str, dict, Task]]
+ List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
+ :param num_fewshot: int
+ Number of examples in few-shot context
+ :param batch_size: int or str, optional
+ Batch size for model
+ :param max_batch_size: int, optional
+ Maximal batch size to try with automatic batch size detection
+ :param device: str, optional
+ PyTorch device (e.g. "cpu" or "cuda:0") for running models
+ :param use_cache: str, optional
+ A path to a sqlite db file for caching model responses. `None` if not caching.
+ :param cache_requests: bool, optional
+ Speed up evaluation by caching the building of dataset requests. `None` if not caching.
+ :param rewrite_requests_cache: bool, optional
+ Rewrites all of the request cache if set to `True`. `None` if not desired.
+ :param delete_requests_cache: bool, optional
+ Deletes all of the request cache if set to `True`. `None` if not desired.
+ :param limit: int or float, optional
+ Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
+ :param bootstrap_iters:
+ Number of iterations for bootstrap statistics, used when calculating stderrs. set to 0 for no stderr calculations to be performed.
+ :param check_integrity: bool
+ Whether to run the relevant part of the test suite for the tasks
+ :param write_out: bool
+ If True, write out an example document and model input for checking task integrity
+ :param log_samples: bool
+ If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
+ :param system_instruction: str
+ System instruction to be applied to the prompt
+ :param apply_chat_template: bool
+ If True, apply chat template to the prompt
+ :param fewshot_as_multiturn: bool
+ Whether to provide the fewshot examples as a multiturn conversation or a single user turn.
+ :param gen_kwargs: str
+ String arguments for model generation
+ Ignored for all tasks with loglikelihood output_type
+ :param predict_only: bool
+ If true only model outputs will be generated and returned. Metrics will not be evaluated
+ :param random_seed: int
+ Random seed for python's random module. If set to None, the seed will not be set.
+ :param numpy_random_seed: int
+ Random seed for numpy. If set to None, the seed will not be set.
+ :param torch_random_seed: int
+ Random seed for torch. If set to None, the seed will not be set.
+ :param fewshot_random_seed: int
+ Random seed for fewshot sampler random generator. If set to None, the seed of generator will be set to None.
+ :param distributed_executor_backend: str
+ The backend to use for distributed execution, `accelerate` or `torchrun`. Defaults to "accelerate" for the `accelerate` library.
+ :return
+ Dictionary of results
+ """
+ seed_message = []
+ if random_seed is not None:
+ # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412
+ seed_message.append(f"Setting random seed to {random_seed}")
+ random.seed(random_seed)
+
+ if numpy_random_seed is not None:
+ seed_message.append(f"Setting numpy seed to {numpy_random_seed}")
+ np.random.seed(numpy_random_seed)
+
+ if torch_random_seed is not None:
+ seed_message.append(f"Setting torch manual seed to {torch_random_seed}")
+ torch.manual_seed(torch_random_seed)
+
+ if seed_message:
+ eval_logger.info(" | ".join(seed_message))
+
+ assert tasks != [], "No tasks specified, or no tasks found. Please verify the task names."
+
+ assert distributed_executor_backend in {"accelerate", "torchrun"}, f"Invalid distributed executor backend: {distributed_executor_backend}. Choose either 'accelerate' or 'torchrun'."
+
+ if gen_kwargs:
+ gen_kwargs = simple_parse_args_string(gen_kwargs)
+ eval_logger.warning(f"generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks.")
+ if gen_kwargs == "":
+ gen_kwargs = None
+
+ if model_args is None:
+ model_args = ""
+
+ if launcher_args is not None:
+ launcher_args = simple_parse_args_string(launcher_args)
+ launcher_name = launcher_args.pop("name")
+ eval_launcher = get_launcher(launcher_name)(**launcher_args)
+ else:
+ eval_launcher = None
+
+ if task_manager is None:
+ task_manager = TaskManager(verbosity, model_name=model)
+
+ if isinstance(model, str):
+ if model_args is None:
+ model_args = ""
+ lm = lmms_eval.models.get_model(model, force_simple).create_from_arg_string(
+ model_args,
+ {
+ "batch_size": batch_size,
+ "max_batch_size": max_batch_size,
+ "device": device,
+ },
+ )
+ elif isinstance(model, lmms_eval.api.model.lmms):
+ lm = model
+ task_type = "simple" if lm.is_simple else "chat"
+ task_dict = get_task_dict(tasks, task_manager, task_type)
+
+ # helper function to recursively apply config overrides to leaf subtasks, skipping their constituent groups.
+ # (setting of num_fewshot ; bypassing metric calculation ; setting fewshot seed)
+ def _adjust_config(task_dict):
+ adjusted_task_dict = {}
+ for task_name, task_obj in task_dict.items():
+ if isinstance(task_obj, dict):
+ adjusted_task_dict = {
+ **adjusted_task_dict,
+ **{task_name: _adjust_config(task_obj)},
+ }
+
+ else:
+ task_obj = task_dict[task_name]
+ if type(task_obj) == tuple:
+ group, task_obj = task_obj
+ if task_obj is None:
+ continue
+ lm.task_dict[task_name] = task_obj.dataset
+ if "generate_until" in task_obj.get_config("output_type"):
+ if gen_kwargs is not None:
+ task_obj.set_config(key="generation_kwargs", value=gen_kwargs, update=True)
+
+ if predict_only:
+ eval_logger.info(f"Processing {task_name} in output-only mode. Metrics will not be calculated!")
+ # we have to change the class properties post-hoc. This is pretty hacky.
+ task_obj.override_metric(metric_name="bypass")
+
+ # override tasks' fewshot values to the provided num_fewshot arg value
+ # except if tasks have it set to 0 manually in their configs--then we should never overwrite that
+ if num_fewshot is not None:
+ if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
+ eval_logger.info(f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored.")
+ else:
+ eval_logger.warning(f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}")
+ task_obj.set_config(key="num_fewshot", value=num_fewshot)
+ else:
+ # if num_fewshot not provided, and the task does not define a default one, default to 0
+ if (default_num_fewshot := task_obj.get_config("num_fewshot")) is None:
+ task_obj.set_config(key="num_fewshot", value=0)
+ # fewshot_random_seed set for tasks, even with a default num_fewshot (e.g. in the YAML file)
+ task_obj.set_fewshot_seed(seed=fewshot_random_seed)
+ # eval_logger.info(f"Setting fewshot random generator seed to {fewshot_random_seed}")
+
+ adjusted_task_dict[task_name] = task_obj
+
+ return adjusted_task_dict
+
+ task_dict = _adjust_config(task_dict)
+
+ if check_integrity:
+ run_task_tests(task_list=tasks)
+
+ if evaluation_tracker is not None:
+ evaluation_tracker.general_config_tracker.log_experiment_args(
+ model_source=model,
+ model_args=model_args,
+ system_instruction=system_instruction,
+ chat_template=lm.chat_template if apply_chat_template else None,
+ fewshot_as_multiturn=fewshot_as_multiturn,
+ )
+
+ # Getting the rank settings
+ local_rank = int(os.environ.get("LOCAL_RANK", 0))
+ global_rank = int(os.environ.get("RANK", 0))
+ world_size = int(os.environ.get("WORLD_SIZE", 1))
+
+ results = evaluate(
+ lm=lm,
+ task_dict=task_dict,
+ limit=limit,
+ cache_requests=cache_requests,
+ rewrite_requests_cache=rewrite_requests_cache,
+ bootstrap_iters=bootstrap_iters,
+ write_out=write_out,
+ log_samples=True if predict_only else log_samples,
+ system_instruction=system_instruction,
+ apply_chat_template=apply_chat_template,
+ fewshot_as_multiturn=fewshot_as_multiturn,
+ verbosity=verbosity,
+ distributed_executor_backend=distributed_executor_backend,
+ cli_args=cli_args,
+ eval_server_launcher=eval_launcher,
+ )
+
+ if global_rank == 0:
+ if isinstance(model, str):
+ model_name = model
+ elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):
+ model_name = model.config._name_or_path
+ else:
+ model_name = type(model).__name__
+
+ # add info about the model and few shot config
+ results["config"] = {
+ "model": model_name,
+ "model_args": model_args,
+ }
+ # add more detailed model info if available TODO: add model info
+ # if isinstance(lm, lmms_eval.models.huggingface.HFLM):
+ # results["config"].update(lm.get_model_info())
+ # add info about execution
+ results["config"].update(
+ {
+ "batch_size": batch_size,
+ "batch_sizes": (list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else []),
+ "device": device,
+ "use_cache": use_cache,
+ "limit": limit,
+ "bootstrap_iters": bootstrap_iters,
+ "gen_kwargs": gen_kwargs,
+ "random_seed": random_seed,
+ "numpy_seed": numpy_random_seed,
+ "torch_seed": torch_random_seed,
+ "fewshot_seed": fewshot_random_seed,
+ }
+ )
+ results["git_hash"] = get_git_commit_hash()
+ results["date"] = datetime_str
+ # add_env_info(results) # additional environment info to results
+ # add_tokenizer_info(results, lm) # additional info about tokenizer
+ return results
+ else:
+ return None
+
+
+decontaminate_suffix = "_decontaminate"
+
+
+@positional_deprecated
+def evaluate(
+ lm: "LM",
+ task_dict,
+ limit: Optional[int] = None,
+ cache_requests: bool = False,
+ rewrite_requests_cache: bool = False,
+ bootstrap_iters: Optional[int] = 100000,
+ write_out: bool = False,
+ log_samples: bool = True,
+ system_instruction: Optional[str] = None,
+ apply_chat_template: bool = False,
+ fewshot_as_multiturn: bool = False,
+ verbosity: str = "INFO",
+ distributed_executor_backend: str = "accelerate",
+ eval_server_launcher: Optional[Union[str, Callable]] = None,
+ cli_args=None,
+):
+ """Instantiate and evaluate a model on a list of tasks.
+
+ :param lm: obj
+ Language Model
+ :param task_dict: dict[str, Task]
+ Dictionary of tasks. Tasks will be taken to have name type(task).config.task .
+ :param limit: int, optional
+ Limit the number of examples per task (only use this for testing)
+ :param bootstrap_iters:
+ Number of iterations for bootstrap statistics, used when calculating stderr. Set to 0 for skipping all stderr calculations.
+ :param write_out: bool
+ If True, write out an example document and model input for checking task integrity
+ :param log_samples: bool
+ If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
+ :param system_instruction: str
+ System instruction to be applied to the prompt
+ :param apply_chat_template: bool
+ If True, apply chat template to the prompt
+ :param fewshot_as_multiturn: bool
+ Whether to provide the fewshot examples as a multiturn conversation or a single user turn.
+ :param distributed_executor_backend: str
+ The backend to use for distributed execution, `accelerate` or `torchrun`. Defaults to "accelerate" for the `accelerate` library.
+ :return
+ Dictionary of results
+ """
+
+ # stores the final result for each task, for each metric/filter pair.
+ results = collections.defaultdict(dict)
+ # Tracks each task's version.
+ versions = collections.defaultdict(dict)
+ # Tracks the YAML configs of all chosen tasks.
+ configs = collections.defaultdict(dict)
+ # logs info about each document evaluated.
+ samples = collections.defaultdict(list)
+ # tracks all Instances/requests a model must generate output on.
+ requests = collections.defaultdict(list)
+ # Aggregated task scores presented with groups
+ results_agg = collections.defaultdict(dict)
+ # Aggregated groups scores only
+ groups_agg = collections.defaultdict(dict)
+ # stores the amount to pad out reqs per req. type so that
+ # number of fwd passes per distributed rank is equal
+ padding_requests = collections.defaultdict(int)
+ # store the hierarchy to do proper ordering
+ task_hierarchy = collections.defaultdict(list)
+ # store the ordering of tasks and groups
+ task_order = collections.defaultdict(int)
+ task_group_alias = collections.defaultdict(dict)
+ # store num-fewshot value per task
+ num_fewshot = collections.defaultdict(int)
+ # Getting the rank settings
+ local_rank = int(os.environ.get("LOCAL_RANK", 0))
+ global_rank = int(os.environ.get("RANK", 0))
+ world_size = int(os.environ.get("WORLD_SIZE", 1))
+ eval_logger.info(f"Running on rank {global_rank} (local rank {local_rank})")
+
+ # get lists of group hierarchy and each type of request
+ eval_tasks = get_task_list(task_dict)
+ name_to_task = {}
+ if not log_samples:
+ if not all("bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys() for task_output in eval_tasks):
+ raise ValueError("log_samples must be True for 'bypass' metric-only tasks")
+
+ if distributed_executor_backend == "accelerate" and not hasattr(lm, "accelerator"):
+ lm.accelerator = Accelerator()
+
+ for task_output in eval_tasks:
+ task: Task = task_output.task
+ task_name = task_output.task_name
+ task.args = cli_args
+
+ name_to_task[task_name] = task
+
+ if type(task) == tuple:
+ group_name, task = task
+ task_hierarchy[group_name].append(task_name)
+ versions[group_name] = "N/A"
+ else:
+ group_name = None
+ task_hierarchy[task_name] = []
+
+ if task is None:
+ continue
+
+ versions[task_name] = task.VERSION
+ configs[task_name] = dict(task.dump_config())
+
+ if "num_fewshot" in configs[task_name]:
+ n_shot = configs[task_name]["num_fewshot"]
+ else:
+ n_shot = 0
+ num_fewshot[task_name] = n_shot
+
+ if "task_alias" in configs[task_name]:
+ task_group_alias[task_name] = configs[task_name]["task_alias"]
+
+ if ("group_alias" in configs[task_name]) and (group_name not in task_group_alias) and (group_name is not None):
+ task_group_alias[group_name] = configs[task_name]["group_alias"]
+
+ limit = get_sample_size(task, limit)
+ task.build_all_requests(
+ limit=limit,
+ rank=global_rank,
+ world_size=world_size,
+ cache_requests=cache_requests, # later we will add them
+ rewrite_requests_cache=rewrite_requests_cache,
+ system_instruction=system_instruction,
+ apply_chat_template=apply_chat_template,
+ fewshot_as_multiturn=fewshot_as_multiturn,
+ chat_template=getattr(lm, "apply_chat_template") if apply_chat_template else None,
+ tokenizer_name=getattr(lm, "tokenizer_name", "") if apply_chat_template else "",
+ )
+ eval_logger.debug(f"Task: {task_output.task_name}; number of requests on this rank: {len(task._instances)}")
+ if write_out:
+ print_writeout(task)
+ # aggregate Instances by LM method requested to get output.
+ for instance in task.instances:
+ reqtype = instance.request_type
+ requests[reqtype].append(instance)
+
+ if world_size > 1:
+ if distributed_executor_backend == "accelerate":
+ instances_rnk = torch.tensor(len(task._instances), device=lm.device)
+ gathered_item = lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist()
+ elif distributed_executor_backend == "torchrun":
+ instances_rnk = torch.tensor(len(task._instances), device=lm.device)
+ gathered_item = torch.zeros(world_size * 1, dtype=instances_rnk.dtype, device=lm.device)
+ dist.all_gather_into_tensor(gathered_item, instances_rnk)
+ gathered_item = gathered_item.cpu().detach().numpy().tolist()
+ else:
+ raise ValueError(f"Invalid distributed_executor_backend: {distributed_executor_backend}. Choose either 'accelerate' or 'torchrun'.")
+
+ # "multiple_choice" task types dispatch (several) "loglikelihood" request types
+ reqtype = "loglikelihood" if task.OUTPUT_TYPE == "multiple_choice" else task.OUTPUT_TYPE
+ # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks)
+ numpad = max(gathered_item) - gathered_item[lm.rank]
+ # todo: may not account for padding in cases like SquadV2 which has multiple req types
+ padding_requests[reqtype] += numpad
+
+ ### Run LMM on inputs, get all outputs ###
+ # execute each type of request
+ for reqtype, reqs in requests.items():
+ eval_logger.info("Running {} requests".format(reqtype))
+ # create `K` copies of each request `req` based off `K = req.repeats`
+ cloned_reqs = []
+ for req in reqs:
+ cloned_reqs.extend([req] * req.repeats)
+
+ if (world_size > 1) and (padding_requests[reqtype] > 0):
+ for _ in range(padding_requests[reqtype]):
+ cloned_reqs.extend([req] * req.repeats)
+
+ # run requests through model
+ resps = getattr(lm, reqtype)(cloned_reqs) # Choiszt run generate until
+
+ # put responses from model into a list of length K for each request.
+ for x, req in zip(resps, cloned_reqs):
+ req.resps.append(x)
+
+ if world_size > 1:
+ if distributed_executor_backend == "accelerate":
+ lm.accelerator.wait_for_everyone()
+ elif distributed_executor_backend == "torchrun":
+ dist.barrier()
+ else:
+ raise ValueError(f"Invalid distributed_executor_backend: {distributed_executor_backend}. Choose either 'accelerate' or 'torchrun'.")
+
+ # Cleaning lm's cuda memory if you are launching llm as judge in local
+ lm.clean()
+ RANK = global_rank
+ WORLD_SIZE = world_size
+ if eval_server_launcher is not None and RANK == 0:
+ eval_server_launcher.launch()
+
+ if world_size > 1:
+ if distributed_executor_backend == "accelerate":
+ lm.accelerator.wait_for_everyone()
+ elif distributed_executor_backend == "torchrun":
+ dist.barrier()
+
+ ### Postprocess outputs ###
+ # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)
+ for task_output in eval_tasks:
+ task = task_output.task
+ task.apply_filters()
+
+ ### Collect values of metrics on all datapoints ###
+ # # unpack results and sort back in order and return control to Task
+ # TODO: make it possible to use a different metric per filter
+ # Pre-process task.instances to group by doc_id
+ instances_by_doc_id = collections.defaultdict(list)
+ for instance in task.instances:
+ instances_by_doc_id[instance.doc_id].append(instance)
+ # Sort instances within each group
+ for instances in instances_by_doc_id.values():
+ instances.sort(key=lambda x: x.idx)
+ # iterate over different filters used
+ for filter_key in task.instances[0].filtered_resps.keys():
+ if cli_args is not None and not cli_args.process_with_media:
+ doc_iterator = create_iterator(enumerate(task.eval_docs_no_media), rank=RANK, limit=int(limit) if limit else None, world_size=WORLD_SIZE)
+ else:
+ doc_iterator = task.doc_iterator(rank=RANK, limit=limit, world_size=WORLD_SIZE)
+ doc_iterator_for_counting = itertools.islice(range(len(task.test_docs())), RANK, limit, WORLD_SIZE) if task.has_test_docs() else itertools.islice(range(len(task.validation_docs())), RANK, limit, WORLD_SIZE)
+ total_docs = sum(1 for _ in doc_iterator_for_counting)
+ pbar = tqdm(total=total_docs, desc=f"Postprocessing", disable=(RANK != 0))
+ for doc_id, doc in doc_iterator:
+ requests = instances_by_doc_id[doc_id]
+ metrics = task.process_results(doc, [req.filtered_resps[filter_key] for req in requests])
+ if log_samples:
+ target = task.doc_to_target(doc)
+ saved_doc = {}
+ for key, value in doc.items():
+ # If image is not in key
+ if "image" not in key:
+ # If audio is also not the value
+ if isinstance(value, dict) and "array" in value:
+ continue
+ else:
+ saved_doc[key] = value
+ filtered_arguments = []
+ for req in requests:
+ # check if req.args is a list of tuples, and each item in the list is a serializable object
+ for value in req.args:
+ if isinstance(value, (str, int, float, bool, list, dict, type(None))):
+ filtered_arguments.append(value)
+ # else:
+ # filtered_arguments.append(_handle_non_serializable(value))
+
+ example = {
+ "doc_id": doc_id,
+ "doc": saved_doc,
+ "target": target,
+ "arguments": filtered_arguments,
+ "resps": [req.resps for req in requests],
+ "filtered_resps": [req.filtered_resps[filter_key] for req in requests],
+ "doc_hash": hash_string(
+ json.dumps(
+ requests[0].doc,
+ indent=2,
+ default=handle_non_serializable,
+ ensure_ascii=False,
+ )
+ ),
+ # Removing prompt hash and target hash here
+ }
+ example.update(metrics)
+ task_output.logged_samples.append(example)
+ for metric, value in metrics.items():
+ task_output.sample_metrics[(metric, filter_key)].append(value)
+ pbar.update(1)
+
+ pbar.close()
+
+ if WORLD_SIZE > 1:
+ # if multigpu, then gather data across all ranks to rank 0
+ # first gather logged samples across all ranks
+ for task_output in eval_tasks:
+ if log_samples:
+ # for task_name, task_samples in list(samples.items()):
+ full_samples = [None] * WORLD_SIZE if RANK == 0 else None
+ per_rank_samples = []
+ for sample in task_output.logged_samples:
+ per_rank_samples.append(sample)
+
+ torch.distributed.gather_object(
+ obj=per_rank_samples,
+ object_gather_list=full_samples,
+ dst=0,
+ )
+
+ if RANK == 0:
+ task_output.logged_samples = list(itertools.chain.from_iterable(full_samples))
+
+ # then collect metrics across all ranks
+ for metrics in task_output.sample_metrics:
+ metric_list = [None] * WORLD_SIZE if RANK == 0 else None
+ torch.distributed.gather_object(
+ obj=task_output.sample_metrics[metrics],
+ object_gather_list=metric_list,
+ dst=0,
+ )
+ if RANK == 0:
+ task_output.sample_metrics[metrics] = list(itertools.chain.from_iterable(metric_list))
+
+ dist.barrier() # Ensure all processes are synced before proceeding
+
+ if RANK == 0:
+ if eval_server_launcher is not None:
+ eval_server_launcher.clean()
+ ### Aggregate results over all datapoints ###
+ # aggregate results ; run bootstrap CIs
+ for task_output in eval_tasks:
+ task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)
+ (
+ results,
+ samples,
+ configs,
+ versions,
+ num_fewshot,
+ higher_is_better,
+ ) = consolidate_results(eval_tasks)
+
+ ### Calculate group metrics ###
+ if bool(results):
+ results, versions, show_group_table, *_ = consolidate_group_results(results, versions, task_dict)
+
+ results_agg, group_agg = prepare_print_tasks(task_dict, results)
+ subtask_list = get_subtask_list(task_dict)
+
+ # collect all higher_is_better values for metrics
+ # in the group's subtasks.
+ # TODO: clean this up ; unify with the below metric_list loop?
+ _higher_is_better = {}
+ for group, task_list in subtask_list.items():
+ if len(task_list) != 0: # subtask list will list "task_name": [] for solo tasks
+ for task in task_list:
+ for m, h in higher_is_better[task].items():
+ if m not in _higher_is_better.keys():
+ _higher_is_better[m] = h
+
+ if m in _higher_is_better and _higher_is_better[m] is not None and _higher_is_better[m] != h:
+ eval_logger.warning(f"Higher_is_better values for metric {m} in group {group} are not consistent. Defaulting to None.")
+ _higher_is_better[m] = None
+ higher_is_better[group] = _higher_is_better
+
+ results_dict = {
+ "results": dict(results_agg.items()),
+ **({"groups": dict(group_agg.items())} if (bool(group_agg) & show_group_table) else {}),
+ "group_subtasks": dict(reversed(subtask_list.items())),
+ "configs": dict(sorted(configs.items())),
+ "versions": dict(sorted(versions.items())),
+ "n-shot": dict(sorted(num_fewshot.items())),
+ "higher_is_better": dict(sorted(higher_is_better.items())),
+ "n-samples": {
+ task_output.task_name: {
+ "original": len(task_output.task.eval_docs),
+ "effective": min(
+ limit if limit else len(task_output.task.eval_docs),
+ len(task_output.task.eval_docs),
+ ),
+ }
+ for task_output in eval_tasks
+ },
+ }
+ if log_samples:
+ results_dict["samples"] = dict(samples)
+ else:
+ results_dict = None
+
+ if WORLD_SIZE > 1:
+ # if muti-gpu, wait for all processes to finish
+ if distributed_executor_backend == "accelerate":
+ # this should work for torchrun as well since it internally calls torch.distributed.barrier()
+ Accelerator().wait_for_everyone()
+ elif distributed_executor_backend == "torchrun":
+ dist.barrier()
+ else:
+ raise ValueError(f"Invalid distributed_executor_backend: {distributed_executor_backend}. Choose either 'accelerate' or 'torchrun'.")
+
+ return results_dict
+
+
+def request_caching_arg_to_dict(cache_requests: str) -> dict:
+ request_caching_args = {
+ "cache_requests": cache_requests in {"true", "refresh"},
+ "rewrite_requests_cache": cache_requests == "refresh",
+ "delete_requests_cache": cache_requests == "delete",
+ }
+
+ return request_caching_args
diff --git a/lmms-eval/lmms_eval/evaluator_utils.py b/lmms-eval/lmms_eval/evaluator_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..1f519066ef367264578d9d45dec237ea5f40d6b7
--- /dev/null
+++ b/lmms-eval/lmms_eval/evaluator_utils.py
@@ -0,0 +1,488 @@
+# credit to https://github.com/EleutherAI/lm-evaluation-harness
+import collections
+import inspect
+import math
+import pathlib
+import sys
+from typing import List, Optional, Tuple, Union
+
+from lmms_eval.api.group import ConfigurableGroup
+from lmms_eval.api.metrics import (
+ aggregate_subtask_metrics,
+ pooled_sample_stderr,
+ stderr_for_metric,
+)
+from lmms_eval.api.task import Task
+from lmms_eval.utils import eval_logger, positional_deprecated
+
+
+class TaskOutput:
+ """
+ Wrapper class for Task outputs.It contains various attributes and methods to manage and calculate metrics for the task.
+
+ Attributes:
+ task (object): The task object.
+ task_name (str): The name of the task.
+ task_config (dict): The configuration of the task.
+ version (str): The version of the task.
+ group_name (str): The name of the task group.
+ n_shot (int): The number of shots for the task.
+ task_alias (str): The alias of the task.
+ group_alias (str): The alias of the task group.
+ is_group (bool): Indicates if the task is a group.
+ logged_samples (list): The list of logged samples.
+ sample_len (int): The length of the samples.
+ sample_metrics (defaultdict): The dictionary of samples' metrics.
+ agg_metrics (defaultdict): The dictionary of aggregate metrics.
+
+ Methods:
+ from_taskdict(cls, task_name: str, task):
+ Creates a TaskOutput instance from a task dictionary.
+
+ calculate_aggregate_metric(bootstrap_iters=100000) -> None:
+ Calculates the aggregate metrics for the task.
+ """
+
+ def __init__(
+ self,
+ task=None,
+ task_name=None,
+ task_config=None,
+ version=None,
+ group_name=None,
+ n_shot=None,
+ task_alias=None,
+ group_alias=None,
+ is_group=None,
+ ):
+ self.task = task
+ self.task_config = task_config
+ self.task_name = task_name
+ self.group_name = group_name
+ self.version = version
+ self.n_shot = n_shot
+ self.task_alias = task_alias
+ self.group_alias = group_alias
+ self.is_group = is_group
+ self.logged_samples = []
+ self.sample_len = None
+ self.sample_metrics = collections.defaultdict(list)
+ self.agg_metrics = collections.defaultdict(list)
+ self.args = None
+
+ @classmethod
+ def from_taskdict(cls, task_name: str, task):
+ if isinstance(task, tuple):
+ group_name, task = task
+ else:
+ group_name = None
+ if not task:
+ # these gets filtered out in get_task_list
+ # once they are added to group hierarchy
+ is_group = True
+ return cls(task=task, task_name=task_name, is_group=is_group, group_name=group_name)
+ version = task.VERSION
+ task_config = dict(task.dump_config())
+ if (n_shot := task_config.get("num_fewshot")) == 0:
+ meta_config = task_config.get("metadata", {})
+ if isinstance(meta_config, dict):
+ n_shot = meta_config.get("num_fewshot", 0)
+ else:
+ eval_logger.info(f"No metadata found in task config for {task_name}, using default n_shot=0")
+ n_shot = 0
+ task_alias = task_config.get("alias")
+ group_alias = task_config.get("group_alias")
+ return cls(
+ task=task,
+ task_name=task_name,
+ task_config=task_config,
+ group_name=group_name,
+ version=version,
+ n_shot=n_shot,
+ task_alias=task_alias,
+ group_alias=group_alias,
+ )
+
+ def calculate_aggregate_metric(self, bootstrap_iters=100000) -> None:
+ for (metric, filter_key), items in self.sample_metrics.items():
+ if metric in self.task.aggregation():
+ agg_fn = self.task.aggregation()[metric]
+ metric_key = f"{metric},{filter_key}"
+ if "args" in inspect.signature(agg_fn).parameters:
+ self.agg_metrics[metric_key] = agg_fn(items, args=self.task.args)
+ else:
+ self.agg_metrics[metric_key] = agg_fn(items)
+ self.sample_len = len(items) # TODO: same sample size for each metric?
+ if isinstance(bootstrap_iters, int):
+ stderr_fn = stderr_for_metric(
+ metric=agg_fn,
+ bootstrap_iters=min(bootstrap_iters, 100) if metric in ["bleu", "chrf", "ter"] else bootstrap_iters,
+ )
+ self.agg_metrics[f"{metric}_stderr,{filter_key}"] = stderr_fn(items) if (stderr_fn and len(items) > 1) else "N/A"
+ else:
+ raise ValueError(f"Received bootstrap_iters '{bootstrap_iters}' but expected an integer. Set to 0 to turn off stderr calculations.")
+
+ def __repr__(self):
+ return f"TaskOutput(task_name={self.task_name}, " f"group_name={self.group_name}, " f"version={self.version}, " f"n_shot={self.n_shot}, " f"task_alias={self.task_alias}, " f"group_alias={self.group_alias})"
+
+
+def get_task_list(task_dict: dict) -> List[TaskOutput]:
+ outputs = []
+ for task_name, task_obj in task_dict.items():
+ if isinstance(task_obj, dict):
+ _outputs = get_task_list(task_obj)
+ outputs.extend(_outputs)
+ else:
+ task_output = TaskOutput.from_taskdict(task_name, task_obj)
+ outputs.append(task_output)
+
+ return outputs
+
+
+def get_subtask_list(task_dict, task_root=None, depth=0):
+ subtask_list = {}
+ for group_obj, task_obj in task_dict.items():
+ if isinstance(group_obj, ConfigurableGroup):
+ # group_name = group_obj.group_name
+ group_name = group_obj.group_name
+ else:
+ group_name = group_obj
+ if isinstance(task_obj, dict):
+ _subtask_list = get_subtask_list(task_obj, task_root=group_name, depth=depth + 1)
+ if task_root:
+ subtask_list.setdefault((task_root, depth), []).extend([_task for (_task, _depth) in _subtask_list.keys() if (_depth - 1) == depth])
+
+ subtask_list = {**subtask_list, **_subtask_list}
+ else:
+ if isinstance(task_obj, ConfigurableGroup):
+ # group_or_task_name = task_obj.group_name
+ group_or_task_name = task_obj.group_name
+ elif isinstance(task_obj, Task):
+ # group_or_task_name = task_obj.task_name
+ group_or_task_name = task_obj.task_name
+
+ if task_root is None:
+ subtask_list.setdefault((group_or_task_name, depth), [])
+ else:
+ subtask_list.setdefault((task_root, depth), []).append(group_or_task_name)
+
+ if depth == 0:
+ _subtask_list = {}
+ for group_key, task_list in subtask_list.items():
+ group_name, depth = group_key
+ _subtask_list[group_name] = task_list
+ subtask_list = _subtask_list
+
+ return subtask_list
+
+
+def print_writeout(task) -> None:
+ for inst in task.instances:
+ # print the prompt for the first few documents
+ if inst.doc_id < 1:
+ eval_logger.info(
+ f"Task: {task}; document {inst.doc_id}; context prompt (starting on next line):\
+ \n{inst.args[0]}\n(end of prompt on previous line)\ntarget string or answer choice index (starting on next line):\n{task.doc_to_target(inst.doc)}\n(end of target on previous line)"
+ )
+ eval_logger.info(f"Request: {str(inst)}")
+
+
+def get_sample_size(task, limit: Optional[int]) -> Union[int, None]:
+ if limit is not None:
+ limit = int(math.ceil(len(task.eval_docs) * limit)) if limit < 1.0 else int(limit)
+ return limit
+
+
+def prepare_print_tasks(
+ task_dict: dict,
+ results: dict,
+ task_depth=0,
+ group_depth=0,
+) -> Tuple[dict, dict]:
+ """
+ @param task_dict: Dictionary representing the group hierarchy of tasks. Each key is a group name and its
+ value is a list of task names.
+ @param results: Dictionary containing the results of each task. Each key is a
+ group name and its value is a dictionary of task results.
+ @param task_depth: The indentation level for printing the task
+ hierarchy. Default is 0.
+ @param group_depth: The indentation level for printing the group
+ hierarchy. Default is 0.
+ @return: A tuple of two dictionaries: results_agg and groups_agg. results_agg contains
+ aggregated results for each task, and groups_agg contains aggregated results for each group.
+
+ Prepares the task hierarchy and aggregates the results for each task and group recursively for printing.
+ """
+
+ def _sort_task_dict(task_dict):
+ """
+ Helper utility. Sorts the task dict at the current level of the hierarchy based on alphabetized task name.
+ Required so that we end up sorting within each sub-header correctly.
+ """
+
+ return dict(
+ sorted(
+ task_dict.items(),
+ key=lambda item: item[0].group_name if isinstance(item[0], ConfigurableGroup) else item[0],
+ )
+ )
+
+ task_agg = collections.defaultdict(dict)
+ group_agg = collections.defaultdict(dict)
+ task_dict = _sort_task_dict(task_dict)
+ for task_or_group_name, task_or_group_obj in task_dict.items():
+ tab_string = " " * task_depth + "- " if task_depth > 0 else ""
+ if isinstance(task_or_group_name, ConfigurableGroup):
+ # string_name = task_or_group_name.group_name
+ name = task_or_group_name.group_name
+ from_configurable_group = True
+ task_or_group_obj = _sort_task_dict(task_or_group_obj)
+ elif isinstance(task_or_group_name, str):
+ name = task_or_group_name
+ if isinstance(task_or_group_obj, Task):
+ # string_name = task_or_group_obj.task_name
+ name = task_or_group_obj.task_name
+ from_configurable_group = False
+
+ task_agg[name] = results[name].copy()
+ if from_configurable_group:
+ if task_or_group_name.group_alias is not None:
+ alias = task_or_group_name.group_alias
+ else:
+ alias = task_or_group_name.group
+ else:
+ if "alias" in task_agg[name]:
+ alias = task_agg[name]["alias"]
+ else:
+ alias = name
+
+ task_agg[name]["alias"] = tab_string + alias
+ if "samples" in task_agg[name]:
+ task_agg[name].pop("samples")
+
+ if from_configurable_group and (" " not in results[name]):
+ group_tab_string = " " * group_depth + "- " if group_depth > 0 else ""
+ group_agg[name] = results[name].copy()
+ group_agg[name]["alias"] = group_tab_string + alias
+ if "samples" in group_agg[name]:
+ group_agg[name].pop("samples")
+
+ if isinstance(task_or_group_obj, dict):
+ task_depth += 1
+ group_depth += 1
+ _task_agg, _group_agg = prepare_print_tasks(task_or_group_obj, results, task_depth, group_depth)
+ task_agg = {
+ **task_agg,
+ **_task_agg,
+ }
+ group_agg = {**group_agg, **_group_agg}
+ task_depth -= 1
+ group_depth -= 1
+ return task_agg, group_agg
+
+
+def consolidate_results(
+ eval_tasks: List[TaskOutput],
+) -> Tuple[dict, dict, dict, dict, dict, dict]:
+ """
+ @param eval_tasks: list(TaskOutput).
+ @return: A tuple containing the consolidated results, samples, configs, versions, and num_fewshot.
+
+ Consolidates the results of multiple evaluation tasks into a single structure.
+
+ The method iterates over each evaluation instance and extracts relevant information to create the consolidated
+ results structure. The consolidated results structure has the following properties:
+
+ - results: A defaultdict with task names as keys and dictionaries as values. Each dictionary contains
+ metric/filter pairs as keys and corresponding metric values as values. The "alias" key is used to store task
+ aliases specified in the task configuration.
+ - samples: A defaultdict with task names as keys and lists of log samples as values.
+ - configs: A defaultdict with task names as keys and task configurations as values.
+ - versions: A defaultdict with task names as keys and task versions as values.
+ - num_fewshot: A defaultdict with task names as keys and number of few-shot samples as values.
+ - higher_is_better: A defaultdict with task names as keys and indicators of whether higher values are better
+ for each metric as values.
+
+ The method then returns the consolidated results, samples, configs, versions, and num_fewshot as a tuple.
+ """
+ # stores the final result for each task, for each metric/filter pair.
+ results = collections.defaultdict(dict)
+ # logs info about each document evaluated.
+ samples = collections.defaultdict(list)
+ # store num-fewshot value per task
+ num_fewshot = collections.defaultdict(int)
+ # Tracks the YAML configs of all chosen task
+ configs = collections.defaultdict(dict)
+ # Tracks each task's version.
+ versions = collections.defaultdict(dict)
+ # Track `higher_is_better` for each metric
+ higher_is_better = collections.defaultdict(dict)
+
+ for task_output in eval_tasks:
+ if "task_alias" in (task_config := task_output.task_config):
+ results[task_output.task_name]["alias"] = task_config["task_alias"]
+ else:
+ results[task_output.task_name]["alias"] = task_output.task_name
+ if group_alias := task_output.group_alias:
+ if group_alias not in results and (group_name := task_output.group_name):
+ results[group_name]["alias"] = group_alias
+ num_fewshot[task_output.task_name] = task_output.n_shot
+ configs[task_output.task_name] = task_output.task_config
+ versions[task_output.task_name] = task_output.version
+ samples[task_output.task_name] = task_output.logged_samples
+ higher_is_better[task_output.task_name] = task_output.task.higher_is_better()
+ for (metric, filter_key), items in task_output.sample_metrics.items():
+ metric_key = f"{metric},{filter_key}"
+ results[task_output.task_name][metric_key] = task_output.agg_metrics[metric_key]
+ results[task_output.task_name]["samples"] = task_output.sample_len
+ results[task_output.task_name][f"{metric}_stderr,{filter_key}"] = task_output.agg_metrics[f"{metric}_stderr,{filter_key}"]
+ return results, samples, configs, versions, num_fewshot, higher_is_better
+
+
+def consolidate_group_results(
+ results,
+ versions,
+ task_dict,
+ task_root=None,
+ show_group_table=False,
+ task_aggregation_list=None,
+) -> Tuple[dict, dict, bool, Union[None, dict]]:
+ """
+ (Recursively) calculates groups' aggregated metrics and updates the results and versions dictionaries with this info.
+
+ @return: a tuple [results, versions, show_group_table, task_aggregation_list] with formats described below:
+
+ - results: A defaultdict with task names (and, after this function is called, group names of
+ groups that perform aggregation) as keys, and dictionaries with "alias" and metric,filter_name pairs as keys.
+ - versions: A defaultdict with task names (and, after this function is called, group names of
+ groups that perform aggregation) as keys, and float values representing the task or group's version if a version is specified. (defaulting to None).
+ - show_group_table: a boolean which is true if there exists a group that requires printing of its aggregated scores in a group table.
+ - task_aggregation_list: a defaultdict listing the subtasks to average over to produce a given group's end metric.
+
+ The method then returns the updated results, versions, show_group_table, and task_aggregation_list as a tuple.
+ In the top-level invocation of this function, task_aggregation_list is ignored.
+ """
+ if task_root is None:
+ task_root = {}
+
+ if task_aggregation_list is None:
+ task_aggregation_list = {}
+
+ for group_or_task, group_or_task_info in task_dict.items():
+ # Convert to string
+ if isinstance(group_or_task, ConfigurableGroup):
+ group_config = group_or_task.config
+ group_or_task = group_or_task.group_name
+ else:
+ group_config = None
+
+ if isinstance(group_or_task_info, Task):
+ if task_root:
+ task_aggregation_list.setdefault(task_root, []).append(group_or_task_info.task_name)
+ else:
+ (
+ results,
+ versions,
+ show_group_table,
+ _task_aggregation_list,
+ ) = consolidate_group_results(
+ results,
+ versions,
+ group_or_task_info,
+ group_or_task,
+ show_group_table,
+ task_aggregation_list,
+ )
+ if task_root:
+ task_aggregation_list.setdefault(task_root, []).extend(task_aggregation_list.get(group_or_task, []))
+
+ if (group_config is None) or (group_config["aggregate_metric_list"] is None):
+ results[group_or_task][" "] = " "
+ continue
+
+ if "aggregate_metric_list" in group_config:
+ agg_metric_list = group_config["aggregate_metric_list"]
+
+ show_group_table = show_group_table | bool(group_config["aggregate_metric_list"])
+
+ task_list = _task_aggregation_list[group_or_task]
+
+ metric_list = list({key for task in task_list for key in results[task].keys() if "_stderr" not in key and key not in ["task", "alias", "samples"]})
+ for metric in metric_list:
+ stderr = "_stderr,".join(metric.split(","))
+
+ # gather metrics, sizes, and stderrs from subtasks
+ metrics = [results[task][metric] for task in task_list if metric in results[task]] # TODO: copy?
+ stderrs = [results[task][stderr] for task in task_list if stderr in results[task]]
+ sizes = [results[task]["samples"] for task in task_list if metric in results[task]]
+
+ for metric_config in agg_metric_list:
+ for filter_name in metric_config["filter_list"]:
+ # if metric != ",".join([metric_config["metric"], filter_name]):
+ # continue
+ if metric_config["metric"] not in metric:
+ continue
+
+ # compute group's pooled metric and stderr
+ if metric_config["aggregation"] == "mean":
+ aggregate_fn = aggregate_subtask_metrics
+ elif callable(metric_config["aggregation"]):
+ aggregate_fn = metric_config["aggregation"]
+ else:
+ raise ValueError(f"Currently, only 'mean' is supported for automatically aggregating scores across groups' subtasks. Got '{metric_config['aggregation']}' for group '{group_or_task}'")
+
+ results[group_or_task][metric] = aggregate_fn(
+ metrics,
+ sizes,
+ metric_config["weight_by_size"],
+ )
+ # TODO: calculate groups' metrics using arbitrary agg fns
+ if "N/A" in stderrs:
+ results[group_or_task][stderr] = "N/A"
+ else:
+ # NOTE: this assumes we are using the mean to aggregate. There are warnings about this elsewhere
+ results[group_or_task][stderr] = pooled_sample_stderr(stderrs, sizes)
+
+ results[group_or_task]["samples"] = sum(sizes)
+ group_metadata = group_config.get("metadata", None)
+ if group_metadata is not None:
+ versions[group_or_task] = group_metadata.get("version", None)
+ # print(results)
+ return results, versions, show_group_table, task_aggregation_list
+
+
+@positional_deprecated
+def find_test_root(start_path: pathlib.Path) -> pathlib.Path:
+ """
+ Search upward in the directory tree to a maximum of three layers
+ to find and return the package root (containing the 'tests' folder)
+ """
+ cur_path = start_path.resolve()
+ max_layers = 3
+ for _ in range(max_layers):
+ if (cur_path / "tests" / "test_version_stable.py").exists():
+ return cur_path
+ else:
+ cur_path = cur_path.parent.resolve()
+ raise FileNotFoundError(f"Unable to find package root within {max_layers} upwards" + f"of {start_path}")
+
+
+@positional_deprecated
+def run_task_tests(task_list: List[str]):
+ """
+ Find the package root and run the tests for the given tasks
+ """
+ import pytest
+
+ package_root = find_test_root(start_path=pathlib.Path(__file__))
+ task_string = " or ".join(task_list)
+ args = [
+ f"{package_root}/tests/test_version_stable.py",
+ f"--rootdir={package_root}",
+ "-k",
+ f"{task_string}",
+ ]
+ sys.path.append(str(package_root))
+ pytest_return_val = pytest.main(args)
+ if pytest_return_val:
+ raise ValueError(f"Not all tests for the specified tasks ({task_list}) ran successfully! Error code: {pytest_return_val}")
diff --git a/lmms-eval/lmms_eval/filters/__init__.py b/lmms-eval/lmms_eval/filters/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..f6c353f523c3b9b7553fbb1ef58417b6544948ac
--- /dev/null
+++ b/lmms-eval/lmms_eval/filters/__init__.py
@@ -0,0 +1,43 @@
+from lmms_eval.api.filter import Filter, FilterEnsemble
+
+from . import extraction, selection, transformation
+
+FILTER_REGISTRY = {
+ "take_first": selection.TakeFirstFilter,
+ "regex": extraction.RegexFilter,
+ "majority_vote": selection.MajorityVoteFilter,
+ "take_first_k": selection.TakeKFilter,
+ "remove_whitespace": extraction.WhitespaceFilter,
+ "lowercase": transformation.LowercaseFilter,
+ "uppercase": transformation.UppercaseFilter,
+ "map": transformation.MapFilter,
+ "multi_choice_regex": extraction.MultiChoiceRegexFilter,
+ # TODO: implement this filter. either it should take in an arbitrary "scoring"/reward function
+ # that takes an input and returns a scalar and then should select the max reward,
+ # or should implement different filters for different ways of handling a reward model's inference.
+ # "arg_max": selection.ArgMaxFilter,
+}
+
+
+def get_filter(filter_name):
+ if filter_name in FILTER_REGISTRY:
+ return FILTER_REGISTRY[filter_name]
+ else:
+ return filter_name
+
+
+def build_filter_ensemble(filter_name, components):
+ """
+ Create a filtering pipeline.
+ """
+ filters = []
+ for function, kwargs in components:
+ if kwargs is None:
+ f = get_filter(function)()
+ else:
+ # create a filter given its name in the registry
+ f = get_filter(function)(**kwargs) # TODO: pass kwargs to filters properly
+ # add the filter as a pipeline step
+ filters.append(f)
+
+ return FilterEnsemble(name=filter_name, filters=filters)
diff --git a/lmms-eval/lmms_eval/filters/decontamination.py b/lmms-eval/lmms_eval/filters/decontamination.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2e1b4504ccbfcf792e3e061de333a24b8c75940
--- /dev/null
+++ b/lmms-eval/lmms_eval/filters/decontamination.py
@@ -0,0 +1,23 @@
+from lmms_eval.api.filter import Filter
+
+
+class DecontaminationFilter(Filter):
+ """
+ A filter which evaluates
+ """
+
+ name = "track_decontamination"
+
+ def __init__(self, path) -> None:
+ """
+
+ TODO: make sure only ever run one time on the train set (should this be cached as a class var? keyed by value for "path").
+ should further cache result on a given (task_name, doc_id)
+ """
+ self._decontam_results = None
+
+ def apply(self, resps, docs) -> None:
+ """
+ Return {"no_contamination", "only_contamination"} keys for the 2 different subsets
+ """
+ pass
diff --git a/lmms-eval/lmms_eval/filters/extraction.py b/lmms-eval/lmms_eval/filters/extraction.py
new file mode 100644
index 0000000000000000000000000000000000000000..9dbc212d6cb713c6ff9d0d5e1cbde96794979d2a
--- /dev/null
+++ b/lmms-eval/lmms_eval/filters/extraction.py
@@ -0,0 +1,282 @@
+import os
+import re
+import sys
+import unicodedata
+
+import openai
+
+from lmms_eval.api.filter import Filter
+
+
+class WhitespaceFilter(Filter):
+ """ """
+
+ def __init__(self) -> None:
+ pass
+
+ def apply(self, resps, docs):
+ def filter_set(inst):
+ filtered_resp = []
+ for resp in inst:
+ if resp.startswith(" "):
+ resp = resp[1:]
+
+ filtered_resp.append(resp)
+
+ return filtered_resp
+
+ filtered_resps = [filter_set(resp) for resp in resps]
+
+ return filtered_resps
+
+
+class RegexFilter(Filter):
+ """ """
+
+ def __init__(
+ self,
+ regex_pattern: str = r"#### (\-?[0-9\.\,]+)",
+ group_select=0,
+ fallback: str = "[invalid]",
+ ) -> None:
+ """
+ pass a string `regex` to run `re.compile(r"regex")` on.
+ `fallback` defines the output returned if no matches for the regex are located.
+ """
+ self.regex_pattern = regex_pattern
+ self.regex = re.compile(regex_pattern)
+ self.group_select = group_select
+ self.fallback = fallback
+
+ def apply(self, resps, docs):
+ # here, we assume we have a list, in which each element is
+ # a list of model responses for some particular input/target pair.
+ # so we process each of these (same input/target response sets)
+ # independently (and keep them a list.)
+ def filter_set(inst):
+ filtered = []
+ for resp in inst:
+ match = self.regex.findall(resp)
+ if match:
+ match = match[self.group_select]
+ if isinstance(match, tuple):
+ match = [m for m in match if m][0]
+ match = match.strip()
+ else:
+ match = self.fallback
+ filtered.append(match)
+ return filtered
+
+ # print(resps)
+ filtered_resps = list(map(lambda x: filter_set(x), resps))
+ # print(filtered_resps)
+
+ return filtered_resps
+
+
+class MultiChoiceRegexFilter(RegexFilter):
+ """
+ A filter used to extract a model's answer on multiple choice questions with
+ letter answers. assumes each document has a "choices" field
+ containing the list of answer choices and that the answer label symbols
+ are of the form (A), (B), (C), ... or A, B, C.
+ """
+
+ def __init__(
+ self,
+ regex_pattern: str = r"#### (\-?[0-9\.\,]+)",
+ group_select=0,
+ fallback: str = "[invalid]",
+ ignore_case=False,
+ ignore_punctuation=False,
+ regexes_to_ignore=None,
+ ) -> None:
+ """
+ regex_pattern: The basic regex pattern to use. If fails to match, we will use the customized match procedure
+ - step 1 : We parse the choices between ([A-Z])s then try to find these choices in the response.
+ - step 2 : We parse the choice with regex :[\s]*([A-?]), where ? varies by number of choices.
+ group_select: Selects the (group_select)th match from the findall result.
+ ignore_case: Ignores the case during step 1 matching
+ ignore_punctuation: Remove the punctuation during step 1 matching
+ regexes_to_ignore: Remove these regexes during step 1 matching
+ """
+ super().__init__(regex_pattern, group_select, fallback)
+ self.ignore_case = ignore_case
+ self.ignore_punctuation = ignore_punctuation
+ self.regexes_to_ignore = regexes_to_ignore
+
+ def apply(self, resps, docs):
+ # here, we assume we have a list, in which each element is
+ # a list of model responses for some particular input/target pair.
+ # so we process each of these (same input/target response sets)
+ # independently (and keep them a list.)
+
+ def find_match(regex, resp, convert_dict={}):
+ match = regex.findall(resp)
+ if match:
+ match = match[self.group_select]
+ if isinstance(match, tuple):
+ match = [m for m in match if m][0]
+ match = match.strip()
+ if match and match in convert_dict:
+ match = convert_dict[match]
+ return match
+
+ punct_tbl = dict.fromkeys(i for i in range(sys.maxunicode) if unicodedata.category(chr(i)).startswith("P"))
+
+ def filter_ignores(st):
+ if self.regexes_to_ignore is not None:
+ for s in self.regexes_to_ignore:
+ st = re.sub(s, "", st)
+
+ if self.ignore_case:
+ st = st.lower()
+
+ if self.ignore_punctuation:
+ # https://stackoverflow.com/a/266162
+ st = st.translate(punct_tbl)
+ return st
+
+ filtered_resps = []
+
+ for r, doc in zip(resps, docs):
+ fallback_regexes = []
+ choice_to_alpha = {}
+ next_alpha = "A"
+
+ without_paren_fallback_regexes = []
+ without_paren_to_target = {}
+
+ choices = doc["choices"]
+ for c in choices:
+ m = filter_ignores(c.strip())
+ fallback_regexes.append(f"{re.escape(m)}")
+ choice_to_alpha[m] = f"({next_alpha})"
+
+ without_paren_fallback_regexes.append(next_alpha)
+ without_paren_to_target[next_alpha] = f"({next_alpha})"
+
+ next_alpha = chr(ord(next_alpha) + 1)
+ fallback_regex = re.compile("|".join(fallback_regexes))
+ without_paren_fallback_regex = "|".join(without_paren_fallback_regexes)
+ without_paren_fallback_regex = re.compile(f":[\s]*({without_paren_fallback_regex})")
+
+ filtered = []
+ for resp in r:
+ match = find_match(self.regex, resp)
+ if not match:
+ match = find_match(fallback_regex, filter_ignores(resp), choice_to_alpha)
+ if not match:
+ match = find_match(without_paren_fallback_regex, resp, without_paren_to_target)
+ if not match:
+ match = self.fallback
+ filtered.append(match)
+ filtered_resps.append(filtered)
+
+ return filtered_resps
+
+
+class ExtendedRegexFilter(RegexFilter):
+ punct_tbl = dict.fromkeys(i for i in range(sys.maxunicode) if unicodedata.category(chr(i)).startswith("P"))
+
+ def __init__(
+ self,
+ regex_pattern: str = r"#### (\-?[0-9\.\,]+)",
+ group_select=0,
+ fallback: str = "[invalid]",
+ ignore_case=False,
+ ignore_punctuation=False,
+ regexes_to_ignore=None,
+ ) -> None:
+ super().__init__(regex_pattern, group_select, fallback)
+ self.ignore_case = ignore_case
+ self.ignore_punctuation = ignore_punctuation
+ self.regexes_to_ignore = regexes_to_ignore
+
+ def filter_ignores(self, st):
+ if self.regexes_to_ignore is not None:
+ for s in self.regexes_to_ignore:
+ st = re.sub(s, "", st)
+
+ if self.ignore_case:
+ st = st.lower()
+
+ if self.ignore_punctuation:
+ # https://stackoverflow.com/a/266162
+ st = st.translate(self.punct_tbl)
+ return st
+
+ def find_match(self, regex, resp, convert_dict={}):
+ match = regex.findall(resp)
+ if match:
+ match = match[self.group_select]
+ if isinstance(match, tuple):
+ match = [m for m in match if m][0]
+ match = match.strip()
+ if match and match in convert_dict:
+ match = convert_dict[match]
+ return match
+
+
+# Designed for the AI2D/RealworldQA dataset
+class SimpleMultiChoiceRegexFilter(ExtendedRegexFilter):
+ def __init__(self, *args, **kwargs):
+ """
+ regex_pattern: The basic regex pattern to use. If fails to match, we will use the customized match procedure
+ - step 1 : We parse the choices between ([A-Z])s then try to find these choices in the response.
+ - step 2 : We parse the choice with regex :[\s]*([A-?]), where ? varies by number of choices.
+ group_select: Selects the (group_select)th match from the findall result.
+ ignore_case: Ignores the case during step 1 matching
+ ignore_punctuation: Remove the punctuation during step 1 matching
+ regexes_to_ignore: Remove these regexes during step 1 matching
+ """
+ super().__init__(*args, **kwargs)
+
+ def apply(self, resps, docs):
+ # here, we assume we have a list, in which each element is
+ # a list of model responses for some particular input/target pair.
+ # so we process each of these (same input/target response sets)
+ # independently (and keep them a list.)
+
+ filtered_resps = []
+
+ for r, doc in zip(resps, docs):
+ fallback_regexes = []
+ choice_to_alpha = {}
+ next_alpha = "A"
+
+ without_paren_fallback_regexes = []
+ without_paren_to_target = {}
+
+ # Regex to extract multiple choice options from the question
+ multiple_choices_regex = re.compile(r"\b([A-Z])\.\s+([^\n]*)")
+ matches = multiple_choices_regex.findall(doc["question"])
+
+ # Build regex patterns and mappings for each choice
+ for m in matches:
+ choice_text = m[1].strip()
+ fallback_regexes.append(f"{re.escape(choice_text)}")
+ choice_to_alpha[choice_text] = next_alpha
+
+ next_alpha = chr(ord(next_alpha) + 1)
+
+ # Compile regex to match any of the extracted choices
+ fallback_regex = re.compile("|".join(fallback_regexes))
+
+ # Process each response
+ filtered = []
+ for resp in r:
+ # Remove any punctuation and extra spaces
+ cleaned_resp = re.sub(r"[^\w\s]", "", resp).strip()
+ # Try to match cleaned response with the choice text
+ match = fallback_regex.search(cleaned_resp)
+ if match and match.group() in choice_to_alpha:
+ # Map the matched choice text back to its corresponding letter
+ filtered.append(choice_to_alpha[match.group()])
+ else:
+ # If no match, return the cleaned response
+ filtered.append(cleaned_resp)
+
+ filtered_resps.append(filtered[0])
+
+ return filtered_resps
diff --git a/lmms-eval/lmms_eval/filters/selection.py b/lmms-eval/lmms_eval/filters/selection.py
new file mode 100644
index 0000000000000000000000000000000000000000..7be1c925c86ea7af934e628d48412ae62e21925c
--- /dev/null
+++ b/lmms-eval/lmms_eval/filters/selection.py
@@ -0,0 +1,59 @@
+from collections import Counter
+
+from lmms_eval.api.filter import Filter
+from lmms_eval.api.registry import register_filter
+
+# TODO: implement "arg_max" filter. either it should take in an arbitrary "scoring"/reward function
+# that takes an input and returns a scalar and then should select the max reward,
+# or should implement different filters for different ways of handling a reward model's inference.
+
+
+@register_filter("take_first")
+class TakeFirstFilter(Filter):
+ def __init__(self) -> None:
+ """
+ Can define custom behavior here, if an individual instantiation of a Filter class should have state.
+ """
+
+ def apply(self, resps, docs):
+ """
+ Assuming each entry of `resps` is a list of model responses, we discard all but the first response.
+ """
+ return map(lambda r: r[0], resps)
+
+
+@register_filter("take_first_k")
+class TakeKFilter(Filter):
+ def __init__(self, **kwargs) -> None:
+ self.k = kwargs.pop("k")
+
+ super().__init__(**kwargs)
+
+ def apply(self, resps, docs):
+ # need resp to be subscriptable to check below
+ resps = list(resps)
+ # check we have at least k responses per doc, else we can't take the first k
+ assert len(resps[0]) >= self.k, f"Need at least {self.k} responses per doc to take first {self.k}, but got {len(resps[0])} only! Please increase TaskConfig.repeats ."
+ selected = map(lambda r: r[: self.k], resps)
+ return selected
+
+
+@register_filter("majority_vote")
+class MajorityVoteFilter(Filter):
+ def __init__(self) -> None:
+ """
+ Can define custom behavior here, if an individual instantiation of a Filter class should have state.
+ """
+
+ def apply(self, resps, docs):
+ """
+ Each entry of `resps` is a list of model responses.
+ We select the response that occurs most frequently in each entry of `resps`.
+ """
+
+ def select_majority(resp):
+ counts = Counter(resp)
+ vote = counts.most_common(1)[0][0]
+ return vote
+
+ return map(lambda r: [select_majority(r)], resps)
diff --git a/lmms-eval/lmms_eval/filters/transformation.py b/lmms-eval/lmms_eval/filters/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..9842115f94a6aaaa95667d27aa6ab3da2dad3bf9
--- /dev/null
+++ b/lmms-eval/lmms_eval/filters/transformation.py
@@ -0,0 +1,48 @@
+from lmms_eval.api.filter import Filter
+
+
+class LowercaseFilter(Filter):
+ def __init__(self) -> None:
+ pass
+
+ def apply(self, resps, docs):
+ def filter_set(inst):
+ return [resp.lower() for resp in inst]
+
+ return [filter_set(resp) for resp in resps]
+
+
+class UppercaseFilter(Filter):
+ def __init__(self) -> None:
+ pass
+
+ def apply(self, resps, docs):
+ def filter_set(inst):
+ return [resp.upper() for resp in inst]
+
+ return [filter_set(resp) for resp in resps]
+
+
+class MapFilter(Filter):
+ def __init__(self, mapping_dict: dict = {}, default_value=None) -> None:
+ """
+ Initializes the MapFilter with a given mapping dictionary and default value.
+
+ Args:
+ - mapping_dict (dict): A dictionary containing the key-value mappings.
+ Default is an empty dictionary.
+ - default_value (Any): The value to be returned when a key is not found in the mapping_dict.
+ Default is None.
+
+ Example:
+ mapper = MapFilter({'A': 1, 'B': 2}, default_value=0)
+ """
+ assert isinstance(mapping_dict, dict), "Provided mapping_dict is not a dictionary"
+ self.mapping_dict = mapping_dict
+ self.default_value = default_value
+
+ def apply(self, resps, docs):
+ def filter_set(inst):
+ return [self.mapping_dict.get(resp, self.default_value) for resp in inst]
+
+ return [filter_set(resp) for resp in resps]
diff --git a/lmms-eval/lmms_eval/llm_judge/__init__.py b/lmms-eval/lmms_eval/llm_judge/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..dd1eab74ee39ba19416524cf502fef5a67e4e28f
--- /dev/null
+++ b/lmms-eval/lmms_eval/llm_judge/__init__.py
@@ -0,0 +1,30 @@
+from .base import ServerInterface
+from .factory import ProviderFactory
+from .protocol import Request, Response, ServerConfig
+from .utils import JudgePromptBuilder, ResponseParser
+
+
+def get_server(server_name: str, config: ServerConfig = None) -> ServerInterface:
+ """
+ Get a server instance by name.
+
+ Args:
+ server_name: Name of the server to instantiate.
+ config: Optional configuration for the server.
+
+ Returns:
+ An instance of ServerInterface.
+ """
+ return ProviderFactory.create_provider(api_type=server_name, config=config)
+
+
+__all__ = [
+ "ServerInterface",
+ "ServerConfig",
+ "Request",
+ "Response",
+ "ProviderFactory",
+ "JudgePromptBuilder",
+ "ResponseParser",
+ "get_server",
+]
diff --git a/lmms-eval/lmms_eval/llm_judge/base.py b/lmms-eval/lmms_eval/llm_judge/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..6f5e10082a01ead92b5fdce8e6b51a67268e3dbd
--- /dev/null
+++ b/lmms-eval/lmms_eval/llm_judge/base.py
@@ -0,0 +1,245 @@
+import abc
+import asyncio
+import json
+import os
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from .protocol import Request, Response, ServerConfig
+from .utils import JudgePromptBuilder, ResponseParser
+
+
+class ServerInterface(abc.ABC):
+ """Abstract base class for judge implementations"""
+
+ def __init__(self, config: Optional[ServerConfig] = None):
+ self.config = config or ServerConfig(model_name="gpt-4")
+
+ @abc.abstractmethod
+ def evaluate(self, request: Request) -> Response:
+ """
+ Evaluate the given request and return a response
+
+ Args:
+ request: JudgeRequest containing the evaluation context
+
+ Returns:
+ JudgeResponse with the evaluation result
+ """
+ pass
+
+ @abc.abstractmethod
+ def is_available(self) -> bool:
+ """Check if the judge service is available"""
+ pass
+
+ def prepare_messages(self, request: Request) -> List[Dict[str, Any]]:
+ """Prepare messages in the format expected by the API"""
+ messages = request.messages.copy()
+
+ # Add system prompt if configured
+ if self.config.system_prompt and not any(m.get("role") == "system" for m in messages):
+ messages.insert(0, {"role": "system", "content": self.config.system_prompt})
+
+ return messages
+
+ def evaluate_binary(self, question: str, answer: str, prediction: str, output_format: str = "0/1", custom_prompt: Optional[str] = None, **kwargs) -> Dict[str, Any]:
+ """Evaluate binary correctness"""
+ # Build prompt
+ prompt = JudgePromptBuilder.build_binary_prompt(question=question, answer=answer, prediction=prediction, output_format=output_format, custom_prompt=custom_prompt, **kwargs)
+
+ # Create request
+ request = Request(messages=[{"role": "user", "content": prompt}], question=question, answer=answer, prediction=prediction, config=self.config)
+
+ # Evaluate
+ response = self.evaluate(request)
+
+ # Parse result
+ parsed_result = ResponseParser.parse_binary_response(response.content, output_format)
+
+ return {"result": parsed_result, "raw_response": response.content, "model": response.model_used, "prompt": prompt, "success": response.success}
+
+ def evaluate_comparative(
+ self, question: str, response1: str, response2: str, context: Optional[str] = None, score_range: Tuple[int, int] = (1, 10), custom_prompt: Optional[str] = None, images: Optional[List[Union[str, bytes]]] = None, **kwargs
+ ) -> Dict[str, Any]:
+ """Evaluate comparative responses"""
+ # Build prompt
+ prompt = JudgePromptBuilder.build_comparative_prompt(question=question, response1=response1, response2=response2, context=context, score_range=score_range, custom_prompt=custom_prompt, **kwargs)
+
+ # Create request
+ request = Request(messages=[{"role": "user", "content": prompt}], question=question, response1=response1, response2=response2, context=context, images=images, config=self.config)
+
+ # Evaluate
+ response = self.evaluate(request)
+
+ # Parse result
+ scores = ResponseParser.parse_comparative_response(response.content)
+
+ return {"scores": scores, "raw_response": response.content, "model": response.model_used, "prompt": prompt, "success": response.success}
+
+ def evaluate_with_rubric(self, question: str, prediction: str, rubric: Dict[str, Any], **kwargs) -> Dict[str, Any]:
+ """Evaluate with a custom rubric"""
+ # Build rubric prompt
+ rubric_text = "\n".join([f"- {k}: {v}" for k, v in rubric.items()])
+
+ prompt = f"""Evaluate the following response according to the given rubric.
+
+Question: {question}
+
+Response: {prediction}
+
+Rubric:
+{rubric_text}
+
+Provide a JSON response with scores for each rubric item."""
+
+ request = Request(messages=[{"role": "user", "content": prompt}], config=self.config)
+
+ # Evaluate
+ response = self.evaluate(request)
+
+ # Parse JSON result
+ parsed_result = ResponseParser.parse_json_response(response.content)
+
+ return {"scores": parsed_result, "raw_response": response.content, "model": response.model_used, "prompt": prompt, "success": response.success}
+
+
+class AsyncServerInterface(ServerInterface):
+ """Abstract base class for async judge implementations"""
+
+ def __init__(self, config: Optional[ServerConfig] = None):
+ super().__init__(config)
+ self.semaphore = asyncio.Semaphore(config.max_concurrent)
+
+ @abc.abstractmethod
+ async def evaluate_async(self, request: Request) -> Response:
+ """
+ Asynchronously evaluate the given request and return a response
+
+ Args:
+ request: JudgeRequest containing the evaluation context
+
+ Returns:
+ JudgeResponse with the evaluation result
+ """
+ pass
+
+ async def evaluate_batch(self, requests: List[Request]) -> List[Response]:
+ """
+ Evaluate multiple requests concurrently
+
+ Args:
+ requests: List of JudgeRequests to evaluate
+
+ Returns:
+ List of JudgeResponses in the same order as requests
+ """
+ tasks = [self.evaluate_async(request) for request in requests]
+ return await asyncio.gather(*tasks)
+
+ def evaluate(self, request: Request) -> Response:
+ """Synchronous wrapper for async evaluation"""
+ loop = asyncio.get_event_loop()
+ return loop.run_until_complete(self.evaluate_async(request))
+
+ async def evaluate_binary_async(self, question: str, answer: str, prediction: str, output_format: str = "0/1", custom_prompt: Optional[str] = None, **kwargs) -> Dict[str, Any]:
+ """Asynchronously evaluate binary correctness"""
+ # Build prompt
+ prompt = JudgePromptBuilder.build_binary_prompt(question=question, answer=answer, prediction=prediction, output_format=output_format, custom_prompt=custom_prompt, **kwargs)
+
+ # Create request
+ request = Request(messages=[{"role": "user", "content": prompt}], question=question, config=self.config)
+
+ # Evaluate
+ response = await self.evaluate_async(request)
+
+ # Parse result
+ parsed_result = ResponseParser.parse_binary_response(response.content, output_format)
+
+ return {"result": parsed_result, "raw_response": response.content, "model": response.model_used, "prompt": prompt, "success": response.success}
+
+ async def evaluate_binary_batch_async(self, questions: List[str], answers: List[str], predictions: List[str], output_format: str = "0/1", custom_prompt: Optional[str] = None, **kwargs) -> List[Dict[str, Any]]:
+ """Asynchronously evaluate multiple binary correctness tasks"""
+ if not (len(questions) == len(answers) == len(predictions)):
+ raise ValueError("All input lists must have the same length")
+
+ tasks = []
+ for q, a, p in zip(questions, answers, predictions):
+ task = self.evaluate_binary_async(q, a, p, output_format, custom_prompt, **kwargs)
+ tasks.append(task)
+
+ return await asyncio.gather(*tasks)
+
+ async def evaluate_comparative_async(
+ self, question: str, response1: str, response2: str, context: Optional[str] = None, score_range: Tuple[int, int] = (1, 10), custom_prompt: Optional[str] = None, images: Optional[List[Union[str, bytes]]] = None, **kwargs
+ ) -> Dict[str, Any]:
+ """Asynchronously evaluate comparative responses"""
+ # Build prompt
+ prompt = JudgePromptBuilder.build_comparative_prompt(question=question, response1=response1, response2=response2, context=context, score_range=score_range, custom_prompt=custom_prompt, **kwargs)
+
+ # Create request
+ request = Request(messages=[{"role": "user", "content": prompt}], question=question, response1=response1, response2=response2, context=context, images=images, config=self.config)
+
+ # Evaluate
+ response = await self.evaluate_async(request)
+
+ # Parse result
+ scores = ResponseParser.parse_comparative_response(response.content)
+
+ return {"scores": scores, "raw_response": response.content, "model": response.model_used, "prompt": prompt, "success": response.success}
+
+ async def evaluate_comparative_batch_async(
+ self,
+ questions: List[str],
+ responses1: List[str],
+ responses2: List[str],
+ contexts: Optional[List[Optional[str]]] = None,
+ score_range: Tuple[int, int] = (1, 10),
+ custom_prompt: Optional[str] = None,
+ images_list: Optional[List[Optional[List[Union[str, bytes]]]]] = None,
+ **kwargs,
+ ) -> List[Dict[str, Any]]:
+ """Asynchronously evaluate multiple comparative response tasks"""
+ if not (len(questions) == len(responses1) == len(responses2)):
+ raise ValueError("Questions and responses lists must have the same length")
+
+ if contexts is None:
+ contexts = [None] * len(questions)
+ if images_list is None:
+ images_list = [None] * len(questions)
+
+ tasks = []
+ for q, r1, r2, ctx, imgs in zip(questions, responses1, responses2, contexts, images_list):
+ task = self.evaluate_comparative_async(q, r1, r2, ctx, score_range, custom_prompt, imgs, **kwargs)
+ tasks.append(task)
+
+ return await asyncio.gather(*tasks)
+
+ async def evaluate_with_rubric_async(self, question: str, prediction: str, rubric: Dict[str, Any], **kwargs) -> Dict[str, Any]:
+ """Asynchronously evaluate with a custom rubric"""
+ # Build rubric prompt
+ rubric_text = "\n".join([f"- {k}: {v}" for k, v in rubric.items()])
+
+ prompt = f"""Evaluate the following response according to the given rubric.
+
+Question: {question}
+
+Response: {prediction}
+
+Rubric:
+{rubric_text}
+
+Provide a JSON response with scores for each rubric item."""
+
+ # Create request with JSON response format
+
+ request = Request(messages=[{"role": "user", "content": prompt}], question=question, prediction=prediction, config=self.config)
+
+ # Evaluate
+ response = await self.evaluate_async(request)
+
+ # Parse JSON result
+ parsed_result = ResponseParser.parse_json_response(response.content)
+
+ return {"scores": parsed_result, "raw_response": response.content, "model": response.model_used, "prompt": prompt, "success": response.success}
diff --git a/lmms-eval/lmms_eval/llm_judge/factory.py b/lmms-eval/lmms_eval/llm_judge/factory.py
new file mode 100644
index 0000000000000000000000000000000000000000..e884bf3b9db9dbdf3643235872a03a242a8cee6a
--- /dev/null
+++ b/lmms-eval/lmms_eval/llm_judge/factory.py
@@ -0,0 +1,54 @@
+import os
+from typing import Optional
+
+from .base import ServerInterface
+from .protocol import ServerConfig
+from .providers import (
+ AsyncAzureOpenAIProvider,
+ AsyncOpenAIProvider,
+ AzureOpenAIProvider,
+ DummyProvider,
+ OpenAIProvider,
+)
+
+
+class ProviderFactory:
+ """Factory for creating judge instances based on configuration"""
+
+ _provider_classes = {"openai": OpenAIProvider, "azure": AzureOpenAIProvider, "async_openai": AsyncOpenAIProvider, "async_azure": AsyncAzureOpenAIProvider, "dummy": DummyProvider}
+
+ # TODO
+ # This should actually be a decorator that registers the class
+ @classmethod
+ def register_additional_providers(cls):
+ """Register additional providers if available"""
+ pass
+
+ @classmethod
+ def create_provider(cls, api_type: Optional[str] = None, config: Optional[ServerConfig] = None) -> ServerInterface:
+ """
+ Create a judge instance based on API type
+
+ Args:
+ api_type: Type of API to use ('openai' or 'azure').
+ If None, will use API_TYPE environment variable
+ config: Configuration for the judge
+
+ Returns:
+ ServerInterface instance
+ """
+ if api_type is None:
+ api_type = os.getenv("API_TYPE", "openai").lower()
+
+ if api_type not in cls._provider_classes:
+ raise ValueError(f"Unknown API type: {api_type}. Supported types: {list(cls._provider_classes.keys())}")
+
+ judge_class = cls._provider_classes[api_type]
+ return judge_class(config=config)
+
+ @classmethod
+ def register_provider(cls, api_type: str, judge_class: type):
+ """Register a new judge implementation"""
+ if not issubclass(judge_class, ServerInterface):
+ raise ValueError(f"{judge_class} must be a subclass of ServerInterface")
+ cls._provider_classes[api_type] = judge_class
diff --git a/lmms-eval/lmms_eval/llm_judge/launcher/__init__.py b/lmms-eval/lmms_eval/llm_judge/launcher/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..b82cd3fff7cccb8d2910a401f2840ad7be319329
--- /dev/null
+++ b/lmms-eval/lmms_eval/llm_judge/launcher/__init__.py
@@ -0,0 +1,36 @@
+import sys
+
+from loguru import logger
+
+logger.remove()
+# Configure logger with detailed format including file path, function name, and line number
+log_format = "{time:YYYY-MM-DD HH:mm:ss} | " "{level: <8} | " "{name}:{function}:{line} - " "{message}"
+logger.add(sys.stdout, level="WARNING", format=log_format)
+
+AVAILABLE_LAUNCHERS = {"sglang": "SGLangLauncher"}
+
+
+def get_launcher(name):
+ """
+ Get a launcher class by its name.
+
+ :param name: The name of the launcher.
+ :return: The launcher class if found, otherwise None.
+ """
+ model_class = AVAILABLE_LAUNCHERS[name]
+ if "." not in model_class:
+ model_class = f"lmms_eval.llm_judge.launcher.{name}.{model_class}"
+
+ try:
+ model_module, model_class = model_class.rsplit(".", 1)
+ module = __import__(model_module, fromlist=[model_class])
+ return getattr(module, model_class)
+ except Exception as e:
+ logger.error(f"Failed to import {model_class} from {name}: {e}")
+ raise
+
+
+__all__ = [
+ "get_launcher",
+ "AVAILABLE_LAUNCHERS",
+]
diff --git a/lmms-eval/lmms_eval/llm_judge/launcher/base.py b/lmms-eval/lmms_eval/llm_judge/launcher/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..a9ba2b78ea78b93019d53b48fabd639d3950766f
--- /dev/null
+++ b/lmms-eval/lmms_eval/llm_judge/launcher/base.py
@@ -0,0 +1,30 @@
+from abc import ABC, abstractmethod
+
+
+class BaseLauncher(ABC):
+ def __init__(self, port: int = 8000, host: str = "localhost", timeout: int = 1200, model: str = "Qwen/Qwen3-8B", **kwargs):
+ super().__init__()
+ self.port = port
+ self.host = host
+ self.timeout = timeout
+ self.model = model
+
+ @abstractmethod
+ def launch(self, *args, **kwargs):
+ """
+ Launch the LLM judge with the given arguments.
+
+ :param args: Positional arguments for the launch.
+ :param kwargs: Keyword arguments for the launch.
+ :return: The result of the launch operation.
+ """
+ pass
+
+ @abstractmethod
+ def clean():
+ """
+ Clean up resources or processes after the launch.
+
+ :return: None
+ """
+ pass
diff --git a/lmms-eval/lmms_eval/llm_judge/launcher/sglang.py b/lmms-eval/lmms_eval/llm_judge/launcher/sglang.py
new file mode 100644
index 0000000000000000000000000000000000000000..144940d8aa699201af16a46c8339f838f768d4ae
--- /dev/null
+++ b/lmms-eval/lmms_eval/llm_judge/launcher/sglang.py
@@ -0,0 +1,42 @@
+from sglang.srt.utils import kill_process_tree
+from sglang.test.test_utils import popen_launch_server
+
+from .base import BaseLauncher
+
+
+class SGLangLauncher(BaseLauncher):
+ def __init__(self, port=8000, host="localhost", timeout=6000, model="Qwen/Qwen3-8B", mem_fraction_static: float = 0.83, tp: int = 8, api_key: str = None, **kwargs):
+ super().__init__(port, host, timeout, model, **kwargs)
+ self.mem_fraction_static = mem_fraction_static
+ self.tp = tp
+ self.base_url = f"http://{self.host}:{self.port}"
+ self.api_key = api_key
+
+ def launch(self, *args, **kwargs):
+ """
+ Launch the SGLang judge with the given arguments.
+
+ :param args: Positional arguments for the launch.
+ :param kwargs: Keyword arguments for the launch.
+ :return: The result of the launch operation.
+ """
+ # Implement the logic to launch SGLang judge
+ other_args = []
+ other_args.extend(["--tensor-parallel-size", str(self.tp)])
+ other_args.extend(["--mem-fraction-static", str(self.mem_fraction_static)])
+ self.process = popen_launch_server(
+ self.model,
+ self.base_url,
+ timeout=self.timeout,
+ api_key=self.api_key,
+ other_args=other_args,
+ )
+
+ def clean(self):
+ """
+ Clean up resources or processes after the SGLang judge launch.
+
+ :return: None
+ """
+ # Implement the cleanup logic for SGLang judge
+ kill_process_tree(self.process.pid)
diff --git a/lmms-eval/lmms_eval/llm_judge/prompt.py b/lmms-eval/lmms_eval/llm_judge/prompt.py
new file mode 100644
index 0000000000000000000000000000000000000000..4a1c392a3c3e7560949a2cec709aa9e523e63993
--- /dev/null
+++ b/lmms-eval/lmms_eval/llm_judge/prompt.py
@@ -0,0 +1,68 @@
+# Default prompts for different judge types
+BINARY_JUDGE_PROMPT = """You are a strict evaluator assessing answer correctness. You must output {positive} for fully correct answers and {negative} for any other case.
+
+# Input
+Question:
+```
+{question}
+```
+Ground Truth Answer:
+```
+{answer}
+```
+Model Prediction:
+```
+{prediction}
+```
+
+# Evaluation Rules
+- The model prediction may contain the reasoning process, you should spot the final answer from it.
+- For multiple-choice questions: Score {positive} if the predicted answer matches the ground truth answer, it can be directly in option letters or the content of the options.
+- For open-ended questions:
+ * Score {positive} if the prediction matches the answer semantically, it can be in different format.
+ * Score {negative} for partially correct answers or answers with extra incorrect information, even if the reasoning process is correct.
+- Ignore minor differences in formatting, capitalization, or spacing since the model may explain in a different way.
+- Treat numerical answers as correct if they match within reasonable precision
+- For questions requiring units, both value and unit must be correct
+
+# Strict Output format
+{positive} or {negative}"""
+
+
+COMPARATIVE_JUDGE_PROMPT = """We would like to request your feedback on the performance of two AI assistants in response to the user question displayed above.
+Please rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of {min_score} to {max_score}, where a higher score indicates better overall performance.
+Please first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.
+In the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment.
+
+[Question]
+{question}
+
+{context_section}
+
+[Assistant 1]
+{response1}
+[End of Assistant 1]
+
+[Assistant 2]
+{response2}
+[End of Assistant 2]
+
+[System]
+{evaluation_instruction}"""
+
+
+CORRECTNESS_JUDGE_PROMPT = """You are given a question, the solution and the correct answer. Please determine if the solution matches the correct answer.
+Focus only on the mathematical or semantic correctness of the content. Ignore any differences in formatting, such as LaTeX syntax, symbols, styles, or additional wrappers (e.g., \\boxed, $...$, or similar). Compare only the core mathematical or textual meaning of the solution and the correct answer.
+The process or reasoning leading to the Solution is irrelevant, ONLY the correctness of the result matters.
+Return only "{positive}" if the solution is correct or "{negative}" if it is incorrect.
+Only return "{positive}" or "{negative}" with no additional text or formatting.
+
+Question:
+{question}
+--------------------------------
+Correct Answer:
+{answer}
+--------------------------------
+Solution:
+{prediction}
+--------------------------------"""
diff --git a/lmms-eval/lmms_eval/llm_judge/protocol.py b/lmms-eval/lmms_eval/llm_judge/protocol.py
new file mode 100644
index 0000000000000000000000000000000000000000..8cdec718bb5316614242555b8b78b751335ae004
--- /dev/null
+++ b/lmms-eval/lmms_eval/llm_judge/protocol.py
@@ -0,0 +1,69 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+# Configuration for retry logic
+DEFAULT_NUM_RETRIES = 5
+DEFAULT_RETRY_DELAY = 10 # seconds
+
+
+@dataclass
+class ServerConfig:
+ """Configuration for judge models"""
+
+ model_name: str
+ temperature: float = 0.0
+ max_tokens: int = 1024
+ top_p: Optional[float] = None
+ timeout: int = 60
+ num_retries: int = DEFAULT_NUM_RETRIES
+ retry_delay: float = DEFAULT_RETRY_DELAY
+ max_concurrent: int = 10 # Maximum concurrent requests
+
+ # Additional config for specific judge tasks
+ system_prompt: Optional[str] = None
+ response_format: Optional[str] = None # 'json' or 'text'
+
+ # Judge-specific parameters
+ judge_type: str = "general" # 'general', 'binary', 'score', 'comparative'
+ output_format: Optional[str] = None # For binary: '0/1' or 'yes/no'
+ score_range: Optional[Tuple[float, float]] = None # For scoring judges
+ evaluation_criteria: Optional[Dict[str, Any]] = None # Custom evaluation criteria
+
+
+@dataclass
+class Request:
+ """Standard request format for judge evaluation"""
+
+ messages: List[Dict[str, Any]]
+ images: Optional[List[Union[str, bytes]]] = None # Image paths or base64 encoded
+ config: Optional[ServerConfig] = None
+
+ # Structured input for specific judge types
+ question: Optional[str] = None
+ answer: Optional[str] = None # Ground truth
+ prediction: Optional[str] = None # Model prediction
+ context: Optional[str] = None # Additional context
+ options: Optional[List[str]] = None # For multiple choice
+
+ # For comparative evaluation
+ response1: Optional[str] = None
+ response2: Optional[str] = None
+
+ # Custom evaluation prompt
+ custom_prompt: Optional[str] = None
+ prompt_kwargs: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class Response:
+ """Standard response format from judge evaluation"""
+
+ content: str
+ model_used: str
+ usage: Optional[Dict[str, int]] = None
+ raw_response: Optional[Any] = None
+
+ # Parsed results for specific judge types
+ parsed_result: Optional[Union[int, float, bool, Tuple[float, float], Dict[str, Any]]] = None
+ success: bool = True
+ error_message: Optional[str] = None
diff --git a/lmms-eval/lmms_eval/llm_judge/providers/__init__.py b/lmms-eval/lmms_eval/llm_judge/providers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..9fbdb284d1f34b63db5d363738e6a5ac965cbaff
--- /dev/null
+++ b/lmms-eval/lmms_eval/llm_judge/providers/__init__.py
@@ -0,0 +1,13 @@
+from .async_azure_openai import AsyncAzureOpenAIProvider
+from .async_openai import AsyncOpenAIProvider
+from .azure_openai import AzureOpenAIProvider
+from .dummy import DummyProvider
+from .openai import OpenAIProvider
+
+__all__ = [
+ "OpenAIProvider",
+ "AzureOpenAIProvider",
+ "AsyncOpenAIProvider",
+ "AsyncAzureOpenAIProvider",
+ "DummyProvider",
+]
diff --git a/lmms-eval/lmms_eval/llm_judge/providers/async_azure_openai.py b/lmms-eval/lmms_eval/llm_judge/providers/async_azure_openai.py
new file mode 100644
index 0000000000000000000000000000000000000000..07d7f335574139d4fd3dcff901309ea58179c1f8
--- /dev/null
+++ b/lmms-eval/lmms_eval/llm_judge/providers/async_azure_openai.py
@@ -0,0 +1,110 @@
+import asyncio
+import os
+from typing import Dict, List, Optional, Union
+
+import aiohttp
+from loguru import logger as eval_logger
+
+from ..base import AsyncServerInterface
+from ..protocol import Request, Response, ServerConfig
+from .openai import OpenAIProvider
+
+
+class AsyncAzureOpenAIProvider(AsyncServerInterface):
+ """Async Azure OpenAI API implementation of the Judge interface"""
+
+ def __init__(self, config: Optional[ServerConfig] = None):
+ super().__init__(config)
+ self.api_key = os.getenv("AZURE_API_KEY", "")
+ self.api_endpoint = os.getenv("AZURE_ENDPOINT", "")
+ self.api_version = os.getenv("API_VERSION", "2024-02-15-preview")
+
+ # Try to use async Azure OpenAI client if available
+ self.use_async_client = False
+ try:
+ from openai import AsyncAzureOpenAI
+
+ self.async_client = AsyncAzureOpenAI(api_key=self.api_key, azure_endpoint=self.api_endpoint, api_version=self.api_version)
+ self.use_async_client = True
+ except ImportError:
+ eval_logger.warning("AsyncAzureOpenAI client not available, using aiohttp")
+
+ def is_available(self) -> bool:
+ return bool(self.api_key and self.api_endpoint)
+
+ async def evaluate_async(self, request: Request) -> Response:
+ """Evaluate using Azure OpenAI API asynchronously"""
+ if not self.is_available():
+ raise ValueError("Azure OpenAI API credentials not configured")
+
+ config = request.config or self.config
+ messages = self.prepare_messages(request)
+
+ # Handle images if present
+ if request.images:
+ messages = self._add_images_to_messages(messages, request.images)
+
+ # Prepare payload
+ payload = {
+ "model": config.model_name,
+ "messages": messages,
+ "temperature": config.temperature,
+ "max_tokens": config.max_tokens,
+ }
+
+ if config.top_p is not None:
+ payload["top_p"] = config.top_p
+
+ if config.response_format == "json":
+ payload["response_format"] = {"type": "json_object"}
+
+ # Make API call with retries
+ async with self.semaphore:
+ for attempt in range(config.num_retries):
+ try:
+ if self.use_async_client:
+ response = await self.async_client.chat.completions.create(**payload)
+ content = response.choices[0].message.content
+ model_used = response.model
+ usage = response.usage.model_dump() if hasattr(response.usage, "model_dump") else None
+ raw_response = response
+ else:
+ response = await self._make_async_request(payload, config.timeout)
+ content = response["choices"][0]["message"]["content"]
+ model_used = response["model"]
+ usage = response.get("usage")
+ raw_response = response
+
+ return Response(content=content.strip(), model_used=model_used, usage=usage, raw_response=raw_response)
+
+ except Exception as e:
+ eval_logger.warning(f"Attempt {attempt + 1}/{config.num_retries} failed: {str(e)}")
+ if attempt < config.num_retries - 1:
+ await asyncio.sleep(config.retry_delay)
+ else:
+ eval_logger.error(f"All {config.num_retries} attempts failed")
+ raise
+
+ async def _make_async_request(self, payload: Dict, timeout: int) -> Dict:
+ """Make async HTTP request to Azure OpenAI API"""
+ headers = {
+ "api-key": self.api_key,
+ "Content-Type": "application/json",
+ }
+
+ # Construct the full URL
+ deployment_name = payload["model"]
+ url = f"{self.api_endpoint}/openai/deployments/{deployment_name}/chat/completions?api-version={self.api_version}"
+
+ async with aiohttp.ClientSession() as session:
+ async with session.post(url, headers=headers, json=payload, timeout=aiohttp.ClientTimeout(total=timeout)) as response:
+ response.raise_for_status()
+ return await response.json()
+
+ def _add_images_to_messages(self, messages: List[Dict], images: List[Union[str, bytes]]) -> List[Dict]:
+ """Add images to messages - reuse from base implementation"""
+ return OpenAIProvider._add_images_to_messages(self, messages, images)
+
+ def _encode_image(self, image_path: str) -> str:
+ """Encode image to base64 - reuse from base implementation"""
+ return OpenAIProvider._encode_image(self, image_path)
diff --git a/lmms-eval/lmms_eval/llm_judge/providers/async_openai.py b/lmms-eval/lmms_eval/llm_judge/providers/async_openai.py
new file mode 100644
index 0000000000000000000000000000000000000000..19df0e599e681408671fa47fcfc06b2becdcb695
--- /dev/null
+++ b/lmms-eval/lmms_eval/llm_judge/providers/async_openai.py
@@ -0,0 +1,105 @@
+import asyncio
+import os
+from typing import Dict, List, Optional, Union
+
+import aiohttp
+from loguru import logger as eval_logger
+
+from ..base import AsyncServerInterface
+from ..protocol import Request, Response, ServerConfig
+from .openai import OpenAIProvider
+
+
+class AsyncOpenAIProvider(AsyncServerInterface):
+ """Async OpenAI API implementation of the Judge interface"""
+
+ def __init__(self, config: Optional[ServerConfig] = None):
+ super().__init__(config)
+ self.api_key = os.getenv("OPENAI_API_KEY", "")
+ self.api_url = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions")
+
+ # Try to use async OpenAI client if available
+ self.use_async_client = False
+ try:
+ from openai import AsyncOpenAI
+
+ self.async_client = AsyncOpenAI(api_key=self.api_key)
+ self.use_async_client = True
+ except ImportError:
+ eval_logger.warning("AsyncOpenAI client not available, using aiohttp")
+
+ def is_available(self) -> bool:
+ return bool(self.api_key)
+
+ async def evaluate_async(self, request: Request) -> Response:
+ """Evaluate using OpenAI API asynchronously"""
+ if not self.is_available():
+ raise ValueError("OpenAI API key not configured")
+
+ config = request.config or self.config
+ messages = self.prepare_messages(request)
+
+ # Handle images if present
+ if request.images:
+ messages = self._add_images_to_messages(messages, request.images)
+
+ # Prepare payload
+ payload = {
+ "model": config.model_name,
+ "messages": messages,
+ "temperature": config.temperature,
+ "max_tokens": config.max_tokens,
+ }
+
+ if config.top_p is not None:
+ payload["top_p"] = config.top_p
+
+ if config.response_format == "json":
+ payload["response_format"] = {"type": "json_object"}
+
+ # Make API call with retries
+ async with self.semaphore:
+ for attempt in range(config.num_retries):
+ try:
+ if self.use_async_client:
+ response = await self.async_client.chat.completions.create(**payload)
+ content = response.choices[0].message.content
+ model_used = response.model
+ usage = response.usage.model_dump() if hasattr(response.usage, "model_dump") else None
+ raw_response = response
+ else:
+ response = await self._make_async_request(payload, config.timeout)
+ content = response["choices"][0]["message"]["content"]
+ model_used = response["model"]
+ usage = response.get("usage")
+ raw_response = response
+
+ return Response(content=content.strip(), model_used=model_used, usage=usage, raw_response=raw_response)
+
+ except Exception as e:
+ eval_logger.warning(f"Attempt {attempt + 1}/{config.num_retries} failed: {str(e)}")
+ if attempt < config.num_retries - 1:
+ await asyncio.sleep(config.retry_delay)
+ else:
+ eval_logger.error(f"All {config.num_retries} attempts failed")
+ raise
+
+ async def _make_async_request(self, payload: Dict, timeout: int) -> Dict:
+ """Make async HTTP request to OpenAI API"""
+ headers = {
+ "Authorization": f"Bearer {self.api_key}",
+ "Content-Type": "application/json",
+ }
+
+ async with aiohttp.ClientSession() as session:
+ async with session.post(self.api_url, headers=headers, json=payload, timeout=aiohttp.ClientTimeout(total=timeout)) as response:
+ response.raise_for_status()
+ return await response.json()
+
+ def _add_images_to_messages(self, messages: List[Dict], images: List[Union[str, bytes]]) -> List[Dict]:
+ """Add images to messages - reuse from base implementation"""
+ return OpenAIProvider._add_images_to_messages(self, messages, images)
+
+ def _encode_image(self, image_path: str) -> str:
+ """Encode image to base64 - reuse from base implementation"""
+ return OpenAIProvider._encode_image(self, image_path)
diff --git a/lmms-eval/lmms_eval/llm_judge/providers/azure_openai.py b/lmms-eval/lmms_eval/llm_judge/providers/azure_openai.py
new file mode 100644
index 0000000000000000000000000000000000000000..184614b810a2a87fe90a6dcc01e3a42283fc8608
--- /dev/null
+++ b/lmms-eval/lmms_eval/llm_judge/providers/azure_openai.py
@@ -0,0 +1,99 @@
+import os
+import time
+from typing import Dict, List, Optional, Union
+
+import requests
+from loguru import logger as eval_logger
+
+from ..protocol import Request, Response, ServerConfig
+from .openai import OpenAIProvider # Import OpenAIJudge for shared methods
+
+
+class AzureOpenAIProvider(OpenAIProvider):
+ """Azure OpenAI implementation of the Judge interface"""
+
+ def __init__(self, config: Optional[ServerConfig] = None):
+ super().__init__(config)
+ self.api_key = os.getenv("AZURE_API_KEY", "")
+ self.api_endpoint = os.getenv("AZURE_ENDPOINT", "")
+ self.api_version = os.getenv("API_VERSION", "2024-02-15-preview")
+
+ # Initialize Azure OpenAI client
+ try:
+ from openai import AzureOpenAI
+
+ self.client = AzureOpenAI(api_key=self.api_key, azure_endpoint=self.api_endpoint, api_version=self.api_version)
+ self.use_client = True
+ except ImportError:
+ eval_logger.warning("Azure OpenAI client not available, falling back to requests")
+ self.use_client = False
+
+ def is_available(self) -> bool:
+ return bool(self.api_key and self.api_endpoint)
+
+ def evaluate(self, request: Request) -> Response:
+ """Evaluate using Azure OpenAI API"""
+ if not self.is_available():
+ raise ValueError("Azure OpenAI API credentials not configured")
+
+ config = request.config or self.config
+ messages = self.prepare_messages(request)
+
+ # Handle images if present
+ if request.images:
+ messages = self._add_images_to_messages(messages, request.images)
+
+ # Prepare payload
+ payload = {
+ "model": config.model_name,
+ "messages": messages,
+ "temperature": config.temperature,
+ "max_tokens": config.max_tokens,
+ }
+
+ if config.top_p is not None:
+ payload["top_p"] = config.top_p
+
+ if config.response_format == "json":
+ payload["response_format"] = {"type": "json_object"}
+
+ # Make API call with retries
+ for attempt in range(config.num_retries):
+ try:
+ if self.use_client:
+ response = self.client.chat.completions.create(**payload)
+ content = response.choices[0].message.content
+ model_used = response.model
+ usage = response.usage.model_dump() if hasattr(response.usage, "model_dump") else None
+ raw_response = response
+ else:
+ response = self._make_request(payload, config.timeout)
+ content = response["choices"][0]["message"]["content"]
+ model_used = response["model"]
+ usage = response.get("usage")
+ raw_response = response
+
+ return Response(content=content.strip(), model_used=model_used, usage=usage, raw_response=raw_response)
+
+ except Exception as e:
+ eval_logger.warning(f"Attempt {attempt + 1}/{config.num_retries} failed: {str(e)}")
+ if attempt < config.num_retries - 1:
+ time.sleep(config.retry_delay)
+ else:
+ eval_logger.error(f"All {config.num_retries} attempts failed")
+ raise
+
+ def _make_request(self, payload: Dict, timeout: int) -> Dict:
+ """Make HTTP request to Azure OpenAI API"""
+ headers = {
+ "api-key": self.api_key,
+ "Content-Type": "application/json",
+ }
+
+ # Construct the full URL
+ deployment_name = payload["model"]
+ url = f"{self.api_endpoint}/openai/deployments/{deployment_name}/chat/completions?api-version={self.api_version}"
+
+ response = requests.post(url, headers=headers, json=payload, timeout=timeout)
+ response.raise_for_status()
+ return response.json()
diff --git a/lmms-eval/lmms_eval/llm_judge/providers/dummy.py b/lmms-eval/lmms_eval/llm_judge/providers/dummy.py
new file mode 100644
index 0000000000000000000000000000000000000000..ac25c10dc58e6a164fb988eb7ad384c115a350b1
--- /dev/null
+++ b/lmms-eval/lmms_eval/llm_judge/providers/dummy.py
@@ -0,0 +1,26 @@
+import base64
+import os
+import time
+from io import BytesIO
+from typing import Dict, List, Optional, Union
+
+import requests
+from loguru import logger as eval_logger
+from PIL import Image
+
+from ..base import ServerInterface
+from ..protocol import Request, Response, ServerConfig
+
+
+class DummyProvider(ServerInterface):
+ """OpenAI API implementation of the Judge interface"""
+
+ def __init__(self, config: Optional[ServerConfig] = None):
+ super().__init__(config)
+
+ def is_available(self) -> bool:
+ return True
+
+ def evaluate(self, request: Request) -> Response:
+ dummy_response = Response(content="dummy", model_used="dummy", usage="dummy", raw_response="dummy")
+ return dummy_response
diff --git a/lmms-eval/lmms_eval/llm_judge/utils.py b/lmms-eval/lmms_eval/llm_judge/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..83621e7535e010c69f87ec47af2937f2863f32be
--- /dev/null
+++ b/lmms-eval/lmms_eval/llm_judge/utils.py
@@ -0,0 +1,115 @@
+import re
+from typing import Any, Dict, Optional, Tuple, Union
+
+from .prompt import (
+ BINARY_JUDGE_PROMPT,
+ COMPARATIVE_JUDGE_PROMPT,
+ CORRECTNESS_JUDGE_PROMPT,
+)
+
+
+class JudgePromptBuilder:
+ """Helper class to build prompts for different judge types"""
+
+ @staticmethod
+ def build_binary_prompt(question: str, answer: str, prediction: str, output_format: str = "0/1", custom_prompt: Optional[str] = None, **kwargs) -> str:
+ """Build prompt for binary evaluation"""
+ if custom_prompt:
+ return custom_prompt.format(question=question, answer=answer, pred=prediction, prediction=prediction, **kwargs)
+
+ positive, negative = ("1", "0") if output_format == "0/1" or output_format == "1/0" else ("Yes", "No")
+
+ return BINARY_JUDGE_PROMPT.format(question=question, answer=answer, prediction=prediction, positive=positive, negative=negative)
+
+ @staticmethod
+ def build_comparative_prompt(
+ question: str, response1: str, response2: str, context: Optional[str] = None, score_range: Tuple[int, int] = (1, 10), custom_prompt: Optional[str] = None, evaluation_instruction: Optional[str] = None, **kwargs
+ ) -> str:
+ """Build prompt for comparative evaluation"""
+ if custom_prompt:
+ return custom_prompt.format(question=question, response1=response1, response2=response2, context=context or "", **kwargs)
+
+ context_section = f"[Context]\n{context}\n\n" if context else ""
+
+ if not evaluation_instruction:
+ evaluation_instruction = f"Please provide scores from {score_range[0]} to {score_range[1]}."
+
+ return COMPARATIVE_JUDGE_PROMPT.format(question=question, response1=response1, response2=response2, context_section=context_section, min_score=score_range[0], max_score=score_range[1], evaluation_instruction=evaluation_instruction)
+
+ @staticmethod
+ def build_correctness_prompt(question: str, answer: str, prediction: str, output_format: str = "yes/no", **kwargs) -> str:
+ """Build prompt for correctness evaluation"""
+ positive, negative = ("Yes", "No") if output_format == "yes/no" else ("1", "0")
+
+ return CORRECTNESS_JUDGE_PROMPT.format(question=question, answer=answer, prediction=prediction, positive=positive, negative=negative)
+
+
+class ResponseParser:
+ """Helper class to parse different types of judge responses"""
+
+ @staticmethod
+ def parse_binary_response(response: str, output_format: str = "0/1") -> Union[int, bool]:
+ """Parse binary response (0/1 or yes/no)"""
+ response = response.strip().lower()
+
+ if output_format == "0/1" or output_format == "1/0":
+ # Check for various formats of 1
+ if any(pattern in response for pattern in ["1", "[1]", "score: 1", "answer: 1"]):
+ return 1
+ else:
+ return 0
+ else:
+ # yes/no format
+ return response == "yes" or response.startswith("yes")
+
+ @staticmethod
+ def parse_score_response(response: str, score_range: Optional[Tuple[float, float]] = None) -> float:
+ """Parse a single score from response"""
+ try:
+ # Try to extract first number from response
+ numbers = re.findall(r"-?\d+(?:\.\d+)?", response)
+ if numbers:
+ score = float(numbers[0])
+ # Clamp to valid range if provided
+ if score_range:
+ score = max(score_range[0], min(score, score_range[1]))
+ return score
+ except Exception as e:
+ pass
+
+ # Return minimum score as default
+ return score_range[0] if score_range else 0.0
+
+ @staticmethod
+ def parse_comparative_response(response: str) -> Tuple[float, float]:
+ """Parse comparative scores from response"""
+ try:
+ # Extract scores from first line
+ lines = response.strip().split("\n")
+ if lines:
+ score_line = lines[0]
+ # Handle different separators
+ score_line = score_line.replace(",", " ").replace(";", " ")
+ scores = re.findall(r"-?\d+(?:\.\d+)?", score_line)
+
+ if len(scores) >= 2:
+ return float(scores[0]), float(scores[1])
+ except Exception as e:
+ pass
+
+ return -1.0, -1.0
+
+ @staticmethod
+ def parse_json_response(response: str) -> Dict[str, Any]:
+ """Parse JSON response"""
+ try:
+ # Try to extract JSON from response
+ json_match = re.search(r"\{.*\}", response, re.DOTALL)
+ if json_match:
+ import json
+
+ return json.loads(json_match.group())
+ except Exception as e:
+ pass
+
+ return {}
diff --git a/lmms-eval/lmms_eval/loggers/__init__.py b/lmms-eval/lmms_eval/loggers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..02b7a6834c6486fde35ef02d715e90be3fba223a
--- /dev/null
+++ b/lmms-eval/lmms_eval/loggers/__init__.py
@@ -0,0 +1,2 @@
+from .evaluation_tracker import EvaluationTracker
+from .wandb_logger import WandbLogger
diff --git a/lmms-eval/lmms_eval/loggers/evaluation_tracker.py b/lmms-eval/lmms_eval/loggers/evaluation_tracker.py
new file mode 100644
index 0000000000000000000000000000000000000000..115feb2d357ef0fdb8b50b2977cca1751c5f799f
--- /dev/null
+++ b/lmms-eval/lmms_eval/loggers/evaluation_tracker.py
@@ -0,0 +1,452 @@
+# code is adapted from https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lmms_eval/loggers/evaluation_tracker.py
+import json
+import os
+import re
+import time
+from collections import defaultdict
+from dataclasses import asdict, dataclass
+from datetime import datetime
+from pathlib import Path
+
+from datasets import load_dataset
+from datasets.utils.metadata import MetadataConfigs
+from huggingface_hub import DatasetCard, DatasetCardData, HfApi, hf_hub_url
+from huggingface_hub.utils import build_hf_headers, get_session, hf_raise_for_status
+
+from lmms_eval.utils import (
+ eval_logger,
+ get_datetime_str,
+ get_file_datetime,
+ get_file_task_name,
+ get_results_filenames,
+ get_sample_results_filenames,
+ handle_non_serializable,
+ hash_string,
+ sanitize_list,
+ sanitize_model_name,
+ sanitize_task_name,
+)
+
+
+@dataclass(init=False)
+class GeneralConfigTracker:
+ """
+ Tracker for the evaluation parameters.
+
+ Attributes:
+ model_source (str): Source of the model (e.g. Hugging Face, GGUF, etc.)
+ model_name (str): Name of the model.
+ model_name_sanitized (str): Sanitized model name for directory creation.
+ start_time (float): Start time of the experiment. Logged at class init.
+ end_time (float): Start time of the experiment. Logged when calling [`GeneralConfigTracker.log_end_time`]
+ total_evaluation_time_seconds (str): Inferred total evaluation time in seconds (from the start and end times).
+ """
+
+ model_source: str = None
+ model_name: str = None
+ model_name_sanitized: str = None
+ system_instruction: str = None
+ system_instruction_sha: str = None
+ fewshot_as_multiturn: bool = None
+ chat_template: str = None
+ chat_template_sha: str = None
+ start_time: float = None
+ end_time: float = None
+ total_evaluation_time_seconds: str = None
+
+ def __init__(self) -> None:
+ """Starts the evaluation timer."""
+ self.start_time = time.perf_counter()
+
+ @staticmethod
+ def _get_model_name(model_args: str) -> str:
+ """Extracts the model name from the model arguments."""
+
+ def extract_model_name(model_args: str, key: str) -> str:
+ """Extracts the model name from the model arguments using a key."""
+ args_after_key = model_args.split(key)[1]
+ return args_after_key.split(",")[0]
+
+ # order does matter, e.g. peft and delta are provided together with pretrained
+ prefixes = ["peft=", "delta=", "pretrained=", "model=", "path=", "engine="]
+ for prefix in prefixes:
+ if prefix in model_args:
+ return extract_model_name(model_args, prefix)
+ return ""
+
+ def log_experiment_args(
+ self,
+ model_source: str,
+ model_args: str,
+ system_instruction: str,
+ chat_template: str,
+ fewshot_as_multiturn: bool,
+ ) -> None:
+ """Logs model parameters and job ID."""
+ self.model_source = model_source
+ self.model_name = GeneralConfigTracker._get_model_name(model_args)
+ self.model_name_sanitized = sanitize_model_name(self.model_name)
+ self.system_instruction = system_instruction
+ self.system_instruction_sha = hash_string(system_instruction) if system_instruction else None
+ self.chat_template = chat_template
+ self.chat_template_sha = hash_string(chat_template) if chat_template else None
+ self.fewshot_as_multiturn = fewshot_as_multiturn
+
+ def log_end_time(self) -> None:
+ """Logs the end time of the evaluation and calculates the total evaluation time."""
+ self.end_time = time.perf_counter()
+ self.total_evaluation_time_seconds = str(self.end_time - self.start_time)
+
+
+class EvaluationTracker:
+ """
+ Keeps track and saves relevant information of the evaluation process.
+ Compiles the data from trackers and writes it to files, which can be published to the Hugging Face hub if requested.
+ """
+
+ def __init__(
+ self,
+ output_path: str = None,
+ hub_results_org: str = "",
+ hub_repo_name: str = "",
+ details_repo_name: str = "",
+ results_repo_name: str = "",
+ push_results_to_hub: bool = False,
+ push_samples_to_hub: bool = False,
+ public_repo: bool = False,
+ token: str = "",
+ leaderboard_url: str = "",
+ point_of_contact: str = "",
+ gated: bool = False,
+ ) -> None:
+ """
+ Creates all the necessary loggers for evaluation tracking.
+
+ Args:
+ output_path (str): Path to save the results. If not provided, the results won't be saved.
+ hub_results_org (str): The Hugging Face organization to push the results to. If not provided, the results will be pushed to the owner of the Hugging Face token.
+ hub_repo_name (str): The name of the Hugging Face repository to push the results to. If not provided, the results will be pushed to `lm-eval-results`.
+ details_repo_name (str): The name of the Hugging Face repository to push the details to. If not provided, the results will be pushed to `lm-eval-results`.
+ result_repo_name (str): The name of the Hugging Face repository to push the results to. If not provided, the results will not be pushed and will be found in the details_hub_repo.
+ push_results_to_hub (bool): Whether to push the results to the Hugging Face hub.
+ push_samples_to_hub (bool): Whether to push the samples to the Hugging Face hub.
+ public_repo (bool): Whether to push the results to a public or private repository.
+ token (str): Token to use when pushing to the Hugging Face hub. This token should have write access to `hub_results_org`.
+ leaderboard_url (str): URL to the leaderboard on the Hugging Face hub on the dataset card.
+ point_of_contact (str): Contact information on the Hugging Face hub dataset card.
+ gated (bool): Whether to gate the repository.
+ """
+ self.general_config_tracker = GeneralConfigTracker()
+
+ self.output_path = output_path
+ self.push_results_to_hub = push_results_to_hub
+ self.push_samples_to_hub = push_samples_to_hub
+ self.public_repo = public_repo
+ self.leaderboard_url = leaderboard_url
+ self.point_of_contact = point_of_contact
+ self.api = HfApi(token=token) if token else None
+ self.gated_repo = gated
+
+ if not self.api and (push_results_to_hub or push_samples_to_hub):
+ raise ValueError("Hugging Face token is not defined, but 'push_results_to_hub' or 'push_samples_to_hub' is set to True. " "Please provide a valid Hugging Face token by setting the HF_TOKEN environment variable.")
+
+ if self.api and hub_results_org == "" and (push_results_to_hub or push_samples_to_hub):
+ hub_results_org = self.api.whoami()["name"]
+ eval_logger.warning(f"hub_results_org was not specified. Results will be pushed to '{hub_results_org}'.")
+
+ if hub_repo_name == "":
+ details_repo_name = details_repo_name if details_repo_name != "" else "lmms-eval-results"
+ results_repo_name = results_repo_name if results_repo_name != "" else details_repo_name
+ else:
+ details_repo_name = hub_repo_name
+ results_repo_name = hub_repo_name
+ eval_logger.warning("hub_repo_name was specified. Both details and results will be pushed to the same repository. Using hub_repo_name is no longer recommended, details_repo_name and results_repo_name should be used instead.")
+
+ self.details_repo = f"{hub_results_org}/{details_repo_name}"
+ self.details_repo_private = f"{hub_results_org}/{details_repo_name}-private"
+ self.results_repo = f"{hub_results_org}/{results_repo_name}"
+ self.results_repo_private = f"{hub_results_org}/{results_repo_name}-private"
+
+ def save_results_aggregated(
+ self,
+ results: dict,
+ samples: dict,
+ datetime_str: str,
+ ) -> None:
+ """
+ Saves the aggregated results and samples to the output path and pushes them to the Hugging Face hub if requested.
+
+ Args:
+ results (dict): The aggregated results to save.
+ samples (dict): The samples results to save.
+ datetime_str (str): The datetime string to use for the results file.
+ """
+ self.general_config_tracker.log_end_time()
+
+ if self.output_path:
+ try:
+ eval_logger.info("Saving results aggregated")
+
+ # calculate cumulative hash for each task - only if samples are provided
+ task_hashes = {}
+ if samples:
+ for task_name, task_samples in samples.items():
+ sample_hashes = [f"sample_{index}" + s["doc_hash"] for index, s in enumerate(task_samples)]
+ task_hashes[task_name] = hash_string("".join(sample_hashes))
+
+ # update initial results dict
+ results.update({"task_hashes": task_hashes})
+ results.update(asdict(self.general_config_tracker))
+ dumped = json.dumps(
+ results,
+ indent=2,
+ default=handle_non_serializable,
+ ensure_ascii=False,
+ )
+
+ path = Path(self.output_path if self.output_path else Path.cwd())
+ path = path.joinpath(self.general_config_tracker.model_name_sanitized)
+ path.mkdir(parents=True, exist_ok=True)
+
+ self.date_id = datetime_str.replace(":", "-")
+ file_results_aggregated = path.joinpath(f"{self.date_id}_results.json")
+ file_results_aggregated.open("w", encoding="utf-8").write(dumped)
+
+ if self.api and self.push_results_to_hub:
+ repo_id = self.results_repo if self.public_repo else self.results_repo_private
+ self.api.create_repo(
+ repo_id=repo_id,
+ repo_type="dataset",
+ private=not self.public_repo,
+ exist_ok=True,
+ )
+ self.api.upload_file(
+ repo_id=repo_id,
+ path_or_fileobj=str(path.joinpath(f"{self.date_id}_results.json")),
+ path_in_repo=os.path.join(
+ self.general_config_tracker.model_name,
+ f"{self.date_id}_results.json",
+ ),
+ repo_type="dataset",
+ commit_message=f"Adding aggregated results for {self.general_config_tracker.model_name}",
+ )
+ eval_logger.info("Successfully pushed aggregated results to the Hugging Face Hub. " f"You can find them at: {repo_id}")
+
+ except Exception as e:
+ eval_logger.warning("Could not save results aggregated")
+ eval_logger.info(repr(e))
+ else:
+ eval_logger.info("Output path not provided, skipping saving results aggregated")
+
+ def save_results_samples(
+ self,
+ task_name: str,
+ samples: dict,
+ ) -> None:
+ """
+ Saves the samples results to the output path and pushes them to the Hugging Face hub if requested.
+
+ Args:
+ task_name (str): The task name to save the samples for.
+ samples (dict): The samples results to save.
+ """
+ if self.output_path:
+ try:
+ path = Path(self.output_path if self.output_path else Path.cwd())
+ path = path.joinpath(self.general_config_tracker.model_name_sanitized)
+ path.mkdir(parents=True, exist_ok=True)
+
+ file_results_samples = path.joinpath(f"{self.date_id}_samples_{task_name}.jsonl")
+
+ for sample in samples:
+ # we first need to sanitize arguments and resps
+ # otherwise we won't be able to load the dataset
+ # using the datasets library
+ # arguments = {}
+ sample["input"] = sample["arguments"][0]
+ sample["resps"] = sanitize_list(sample["resps"])
+ sample["filtered_resps"] = sanitize_list(sample["filtered_resps"])
+ if sample["filtered_resps"] == sample["resps"][0] or sample["filtered_resps"] == sample["resps"]:
+ sample.pop("resps")
+ sample["target"] = str(sample["target"])
+ sample.pop("arguments")
+ sample.pop("doc")
+
+ sample_dump = (
+ json.dumps(
+ sample,
+ default=handle_non_serializable,
+ ensure_ascii=False,
+ )
+ + "\n"
+ )
+
+ with open(file_results_samples, "a", encoding="utf-8") as f:
+ f.write(sample_dump)
+
+ eval_logger.info(f"Saving samples to {file_results_samples}")
+
+ if self.api and self.push_samples_to_hub:
+ repo_id = self.details_repo if self.public_repo else self.details_repo_private
+ self.api.create_repo(
+ repo_id=repo_id,
+ repo_type="dataset",
+ private=not self.public_repo,
+ exist_ok=True,
+ )
+ try:
+ if self.gated_repo:
+ headers = build_hf_headers()
+ r = get_session().put(
+ url=f"https://huggingface.co/api/datasets/{repo_id}/settings",
+ headers=headers,
+ json={"gated": "auto"},
+ )
+ hf_raise_for_status(r)
+ except Exception as e:
+ eval_logger.warning("Could not gate the repository")
+ eval_logger.info(repr(e))
+ self.api.upload_folder(
+ repo_id=repo_id,
+ folder_path=str(path),
+ path_in_repo=self.general_config_tracker.model_name_sanitized,
+ repo_type="dataset",
+ commit_message=f"Adding samples results for {task_name} to {self.general_config_tracker.model_name}",
+ )
+ eval_logger.info(f"Successfully pushed sample results for task: {task_name} to the Hugging Face Hub. " f"You can find them at: {repo_id}")
+
+ except Exception as e:
+ eval_logger.warning("Could not save sample results")
+ eval_logger.info(repr(e))
+ else:
+ eval_logger.info("Output path not provided, skipping saving sample results")
+
+ def recreate_metadata_card(self) -> None:
+ """
+ Creates a metadata card for the evaluation results dataset and pushes it to the Hugging Face hub.
+ """
+
+ eval_logger.info("Recreating metadata card")
+ repo_id = self.details_repo if self.public_repo else self.details_repo_private
+
+ files_in_repo = self.api.list_repo_files(repo_id=repo_id, repo_type="dataset")
+ results_files = get_results_filenames(files_in_repo)
+ sample_files = get_sample_results_filenames(files_in_repo)
+
+ # Build a dictionary to store the latest evaluation datetime for:
+ # - Each tested model and its aggregated results
+ # - Each task and sample results, if existing
+ # i.e. {
+ # "org__model_name__gsm8k": "2021-09-01T12:00:00",
+ # "org__model_name__ifeval": "2021-09-01T12:00:00",
+ # "org__model_name__results": "2021-09-01T12:00:00"
+ # }
+ latest_task_results_datetime = defaultdict(lambda: datetime.min.isoformat())
+
+ for file_path in sample_files:
+ file_path = Path(file_path)
+ filename = file_path.name
+ model_name = file_path.parent
+ task_name = get_file_task_name(filename)
+ results_datetime = get_file_datetime(filename)
+ task_name_sanitized = sanitize_task_name(task_name)
+ # Results and sample results for the same model and task will have the same datetime
+ samples_key = f"{model_name}__{task_name_sanitized}"
+ results_key = f"{model_name}__results"
+ latest_datetime = max(
+ latest_task_results_datetime[samples_key],
+ results_datetime,
+ )
+ latest_task_results_datetime[samples_key] = latest_datetime
+ latest_task_results_datetime[results_key] = max(
+ latest_task_results_datetime[results_key],
+ latest_datetime,
+ )
+
+ # Create metadata card
+ card_metadata = MetadataConfigs()
+
+ # Add the latest aggregated results to the metadata card for easy access
+ for file_path in results_files:
+ file_path = Path(file_path)
+ results_filename = file_path.name
+ model_name = file_path.parent
+ eval_date = get_file_datetime(results_filename)
+ eval_date_sanitized = re.sub(r"[^\w\.]", "_", eval_date)
+ results_filename = Path("**") / Path(results_filename).name
+ config_name = f"{model_name}__results"
+ sanitized_last_eval_date_results = re.sub(r"[^\w\.]", "_", latest_task_results_datetime[config_name])
+
+ if eval_date_sanitized == sanitized_last_eval_date_results:
+ # Ensure that all results files are listed in the metadata card
+ current_results = card_metadata.get(config_name, {"data_files": []})
+ current_results["data_files"].append({"split": eval_date_sanitized, "path": [str(results_filename)]})
+ card_metadata[config_name] = current_results
+ # If the results file is the newest, update the "latest" field in the metadata card
+ card_metadata[config_name]["data_files"].append({"split": "latest", "path": [str(results_filename)]})
+
+ # Add the tasks details configs
+ for file_path in sample_files:
+ file_path = Path(file_path)
+ filename = file_path.name
+ model_name = file_path.parent
+ task_name = get_file_task_name(filename)
+ eval_date = get_file_datetime(filename)
+ task_name_sanitized = sanitize_task_name(task_name)
+ eval_date_sanitized = re.sub(r"[^\w\.]", "_", eval_date)
+ results_filename = Path("**") / Path(filename).name
+ config_name = f"{model_name}__{task_name_sanitized}"
+ sanitized_last_eval_date_results = re.sub(r"[^\w\.]", "_", latest_task_results_datetime[config_name])
+ if eval_date_sanitized == sanitized_last_eval_date_results:
+ # Ensure that all sample results files are listed in the metadata card
+ current_details_for_task = card_metadata.get(config_name, {"data_files": []})
+ current_details_for_task["data_files"].append({"split": eval_date_sanitized, "path": [str(results_filename)]})
+ card_metadata[config_name] = current_details_for_task
+ # If the samples results file is the newest, update the "latest" field in the metadata card
+ card_metadata[config_name]["data_files"].append({"split": "latest", "path": [str(results_filename)]})
+
+ # Get latest results and extract info to update metadata card examples
+ latest_datetime = max(latest_task_results_datetime.values())
+ latest_model_name = max(latest_task_results_datetime, key=lambda k: latest_task_results_datetime[k])
+ last_results_file = [f for f in results_files if latest_datetime.replace(":", "-") in f][0]
+ last_results_file_path = hf_hub_url(repo_id=repo_id, filename=last_results_file, repo_type="dataset")
+ latest_results_file = load_dataset("json", data_files=last_results_file_path, split="train")
+ results_dict = latest_results_file["results"][0]
+ new_dictionary = {"all": results_dict}
+ new_dictionary.update(results_dict)
+ results_string = json.dumps(new_dictionary, indent=4)
+
+ dataset_summary = "Dataset automatically created during the evaluation run of model "
+ if self.general_config_tracker.model_source == "hf":
+ dataset_summary += f"[{self.general_config_tracker.model_name}](https://huggingface.co/{self.general_config_tracker.model_name})\n"
+ else:
+ dataset_summary += f"{self.general_config_tracker.model_name}\n"
+ dataset_summary += (
+ f"The dataset is composed of {len(card_metadata)-1} configuration(s), each one corresponding to one of the evaluated task.\n\n"
+ f"The dataset has been created from {len(results_files)} run(s). Each run can be found as a specific split in each "
+ 'configuration, the split being named using the timestamp of the run.The "train" split is always pointing to the latest results.\n\n'
+ 'An additional configuration "results" store all the aggregated results of the run.\n\n'
+ "To load the details from a run, you can for instance do the following:\n"
+ )
+ if self.general_config_tracker.model_source == "hf":
+ dataset_summary += "```python\nfrom datasets import load_dataset\n" f'data = load_dataset(\n\t"{repo_id}",\n\tname="{latest_model_name}",\n\tsplit="latest"\n)\n```\n\n'
+ dataset_summary += (
+ "## Latest results\n\n"
+ f'These are the [latest results from run {latest_datetime}]({last_results_file_path.replace("/resolve/", "/blob/")}) '
+ "(note that there might be results for other tasks in the repos if successive evals didn't cover the same tasks. "
+ 'You find each in the results and the "latest" split for each eval):\n\n'
+ f"```python\n{results_string}\n```"
+ )
+ card_data = DatasetCardData(
+ dataset_summary=dataset_summary,
+ repo_url=f"https://huggingface.co/{self.general_config_tracker.model_name}",
+ pretty_name=f"Evaluation run of {self.general_config_tracker.model_name}",
+ leaderboard_url=self.leaderboard_url,
+ point_of_contact=self.point_of_contact,
+ )
+ card_metadata.to_dataset_card_data(card_data)
+ card = DatasetCard.from_template(
+ card_data,
+ pretty_name=card_data.pretty_name,
+ )
+ card.push_to_hub(repo_id, repo_type="dataset")
diff --git a/lmms-eval/lmms_eval/loggers/utils.py b/lmms-eval/lmms_eval/loggers/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..024611da72911be33236a26cf1061d61b4ada974
--- /dev/null
+++ b/lmms-eval/lmms_eval/loggers/utils.py
@@ -0,0 +1,136 @@
+import os
+import pickle
+import re
+import subprocess
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple, Union
+
+import numpy as np
+from loguru import logger
+from torch.utils.collect_env import get_pretty_env_info
+from transformers import __version__ as trans_version
+
+
+def remove_none_pattern(input_string: str) -> Tuple[str, bool]:
+ """Remove the ',none' substring from the input_string if it exists at the end.
+
+ Args:
+ input_string (str): The input string from which to remove the ',none' substring.
+
+ Returns:
+ Tuple[str, bool]: A tuple containing the modified input_string with the ',none' substring removed
+ and a boolean indicating whether the modification was made (True) or not (False).
+ """
+ # Define the pattern to match ',none' at the end of the string
+ pattern = re.compile(r",none$")
+
+ # Use sub() to replace ',none' with an empty string
+ result = re.sub(pattern, "", input_string)
+
+ # check if the input_string changed
+ removed = result != input_string
+
+ return result, removed
+
+
+def is_serializable(o: Any) -> bool:
+ try:
+ pickle.dumps(o)
+ return True
+ except (pickle.PickleError, TypeError, AttributeError):
+ return False
+
+
+def _handle_non_serializable(o: Any) -> Union[int, str, list]:
+ """Handle non-serializable objects by converting them to serializable types.
+
+ Args:
+ o (Any): The object to be handled.
+
+ Returns:
+ Union[int, str, list]: The converted object. If the object is of type np.int64 or np.int32,
+ it will be converted to int. If the object is of type set, it will be converted
+ to a list. Otherwise, it will be converted to str.
+ """
+ if isinstance(o, np.int64) or isinstance(o, np.int32):
+ return int(o)
+ elif isinstance(o, set):
+ return list(o)
+ else:
+ return str(o)
+
+
+def get_commit_from_path(repo_path: Union[Path, str]) -> Optional[str]:
+ try:
+ git_folder = Path(repo_path, ".git")
+ if git_folder.is_file():
+ git_folder = Path(
+ git_folder.parent,
+ git_folder.read_text(encoding="utf-8").split("\n")[0].split(" ")[-1],
+ )
+ if Path(git_folder, "HEAD").exists():
+ head_name = Path(git_folder, "HEAD").read_text(encoding="utf-8").split("\n")[0].split(" ")[-1]
+ head_ref = Path(git_folder, head_name)
+ git_hash = head_ref.read_text(encoding="utf-8").replace("\n", "")
+ else:
+ git_hash = None
+ except Exception as err:
+ logger.debug(f"Failed to retrieve a Git commit hash from path: {str(repo_path)}. Error: {err}")
+ return None
+ return git_hash
+
+
+def get_git_commit_hash():
+ """
+ Gets the git commit hash of your current repo (if it exists).
+ Source: https://github.com/EleutherAI/gpt-neox/blob/b608043be541602170bfcfb8ec9bf85e8a0799e0/megatron/neox_arguments/neox_args.py#L42
+ """
+ try:
+ git_hash = subprocess.check_output(["git", "describe", "--always"]).strip()
+ git_hash = git_hash.decode()
+ except (subprocess.CalledProcessError, FileNotFoundError):
+ # FileNotFoundError occurs when git not installed on system
+ git_hash = get_commit_from_path(os.getcwd()) # git hash of repo if exists
+ return git_hash
+
+
+def add_env_info(storage: Dict[str, Any]):
+ try:
+ pretty_env_info = get_pretty_env_info()
+ except Exception as err:
+ pretty_env_info = str(err)
+ transformers_version = trans_version
+ upper_dir_commit = get_commit_from_path(Path(os.getcwd(), "..")) # git hash of upper repo if exists
+ added_info = {
+ "pretty_env_info": pretty_env_info,
+ "transformers_version": transformers_version,
+ "upper_git_hash": upper_dir_commit, # in case this repo is submodule
+ }
+ storage.update(added_info)
+
+
+def add_tokenizer_info(storage: Dict[str, Any], lm):
+ if getattr(lm, "tokenizer", False):
+ try:
+ tokenizer_info = {
+ "tokenizer_pad_token": [
+ lm.tokenizer.pad_token,
+ str(lm.tokenizer.pad_token_id),
+ ],
+ "tokenizer_eos_token": [
+ lm.tokenizer.eos_token,
+ str(lm.tokenizer.eos_token_id),
+ ],
+ "tokenizer_bos_token": [
+ lm.tokenizer.bos_token,
+ str(lm.tokenizer.bos_token_id),
+ ],
+ "eot_token_id": getattr(lm, "eot_token_id", None),
+ "max_length": getattr(lm, "max_length", None),
+ }
+ storage.update(tokenizer_info)
+ except Exception as err:
+ logger.debug(f"Logging detailed tokenizer info failed with {err}, skipping...")
+ # seems gguf and textsynth do not have tokenizer
+ else:
+ logger.debug("LM does not have a 'tokenizer' attribute, not logging tokenizer metadata to results.")
diff --git a/lmms-eval/lmms_eval/loggers/wandb_logger.py b/lmms-eval/lmms_eval/loggers/wandb_logger.py
new file mode 100644
index 0000000000000000000000000000000000000000..b30b1ced5135a929cc8fe02430a10c529e8b77b1
--- /dev/null
+++ b/lmms-eval/lmms_eval/loggers/wandb_logger.py
@@ -0,0 +1,315 @@
+import copy
+import json
+import logging
+from typing import Any, Dict, List, Literal, Tuple
+
+import numpy as np
+import pandas as pd
+from loguru import logger
+from packaging.version import Version
+
+from lmms_eval.loggers.utils import _handle_non_serializable, remove_none_pattern
+
+
+def get_wandb_printer() -> Literal["Printer"]:
+ """Returns a wandb printer instance for pretty stdout."""
+ from wandb.sdk.lib.printer import get_printer
+ from wandb.sdk.wandb_settings import Settings
+
+ printer = get_printer(Settings()._jupyter)
+ return printer
+
+
+class WandbLogger:
+ def __init__(self, **kwargs) -> None:
+ """Attaches to wandb logger if already initialized. Otherwise, passes kwargs to wandb.init()
+
+ Args:
+ kwargs Optional[Any]: Arguments for configuration.
+
+ Parse and log the results returned from evaluator.simple_evaluate() with:
+ wandb_logger.post_init(results)
+ wandb_logger.log_eval_result()
+ wandb_logger.log_eval_samples(results["samples"])
+ """
+ try:
+ import wandb
+
+ assert Version(wandb.__version__) >= Version("0.13.6")
+ if Version(wandb.__version__) < Version("0.13.6"):
+ wandb.require("report-editing:v0")
+ except Exception as e:
+ logger.warning("To use the wandb reporting functionality please install wandb>=0.13.6.\n" "To install the latest version of wandb run `pip install wandb --upgrade`\n" f"{e}")
+
+ self.wandb_args: Dict[str, Any] = kwargs
+
+ # initialize a W&B run
+ if wandb.run is None:
+ self.run = wandb.init(**self.wandb_args)
+ else:
+ self.run = wandb.run
+
+ self.printer = get_wandb_printer()
+
+ def post_init(self, results: Dict[str, Any]) -> None:
+ self.results: Dict[str, Any] = copy.deepcopy(results)
+ self.task_names: List[str] = list(results.get("results", {}).keys())
+ self.group_names: List[str] = list(results.get("groups", {}).keys())
+
+ def _get_config(self) -> Dict[str, Any]:
+ """Get configuration parameters."""
+ self.task_configs = self.results.get("configs", {})
+ cli_configs = self.results.get("config", {})
+ configs = {
+ "task_configs": self.task_configs,
+ "cli_configs": cli_configs,
+ }
+
+ return configs
+
+ def _sanitize_results_dict(self) -> Tuple[Dict[str, str], Dict[str, Any]]:
+ """Sanitize the results dictionary."""
+ _results = copy.deepcopy(self.results.get("results", dict()))
+
+ # Remove None from the metric string name
+ tmp_results = copy.deepcopy(_results)
+ for task_name in self.task_names:
+ task_result = tmp_results.get(task_name, dict())
+ for metric_name, metric_value in task_result.items():
+ _metric_name, removed = remove_none_pattern(metric_name)
+ if removed:
+ _results[task_name][_metric_name] = metric_value
+ _results[task_name].pop(metric_name)
+
+ # remove string valued keys from the results dict
+ wandb_summary = {}
+ for task in self.task_names:
+ task_result = _results.get(task, dict())
+ for metric_name, metric_value in task_result.items():
+ if isinstance(metric_value, str):
+ wandb_summary[f"{task}/{metric_name}"] = metric_value
+
+ for summary_metric, summary_value in wandb_summary.items():
+ _task, _summary_metric = summary_metric.split("/")
+ _results[_task].pop(_summary_metric)
+
+ tmp_results = copy.deepcopy(_results)
+ for task_name, task_results in tmp_results.items():
+ for metric_name, metric_value in task_results.items():
+ _results[f"{task_name}/{metric_name}"] = metric_value
+ _results[task_name].pop(metric_name)
+ for task in self.task_names:
+ _results.pop(task)
+
+ return wandb_summary, _results
+
+ def _log_results_as_table(self) -> None:
+ """Generate and log evaluation results as a table to W&B."""
+ columns = [
+ "Version",
+ "Filter",
+ "num_fewshot",
+ "Metric",
+ "Value",
+ "Stderr",
+ ]
+
+ def make_table(columns: List[str], key: str = "results"):
+ import wandb
+
+ table = wandb.Table(columns=columns)
+ results = copy.deepcopy(self.results)
+
+ for k, dic in results.get(key).items():
+ if k in self.group_names and not key == "groups":
+ continue
+ version = results.get("versions").get(k)
+ if version == "N/A":
+ version = None
+ n = results.get("n-shot").get(k)
+
+ for (mf), v in dic.items():
+ m, _, f = mf.partition(",")
+ if m.endswith("_stderr"):
+ continue
+ if m == "alias":
+ continue
+
+ if m + "_stderr" + "," + f in dic:
+ se = dic[m + "_stderr" + "," + f]
+ if se != "N/A":
+ se = "%.4f" % se
+ table.add_data(*[k, version, f, n, m, str(v), str(se)])
+ else:
+ table.add_data(*[k, version, f, n, m, str(v), ""])
+
+ return table
+
+ # log the complete eval result to W&B Table
+ table = make_table(["Tasks"] + columns, "results")
+ self.run.log({"evaluation/eval_results": table})
+
+ if "groups" in self.results.keys():
+ table = make_table(["Groups"] + columns, "groups")
+ self.run.log({"evaluation/group_eval_results": table})
+
+ def _log_results_as_artifact(self) -> None:
+ """Log results as JSON artifact to W&B."""
+ import wandb
+
+ dumped = json.dumps(self.results, indent=2, default=_handle_non_serializable, ensure_ascii=False)
+ artifact = wandb.Artifact("results", type="eval_results")
+ with artifact.new_file("results.json", mode="w", encoding="utf-8") as f:
+ f.write(dumped)
+ self.run.log_artifact(artifact)
+
+ def log_eval_result(self) -> None:
+ """Log evaluation results to W&B."""
+ # Log configs to wandb
+ configs = self._get_config()
+ self.run.config.update(configs)
+
+ wandb_summary, self.wandb_results = self._sanitize_results_dict()
+ # update wandb.run.summary with items that were removed
+ self.run.summary.update(wandb_summary)
+ # Log the evaluation metrics to wandb
+ self.run.log(self.wandb_results)
+ # Log the evaluation metrics as W&B Table
+ self._log_results_as_table()
+ # Log the results dict as json to W&B Artifacts
+ self._log_results_as_artifact()
+
+ def _generate_dataset(self, data: List[Dict[str, Any]], config: Dict[str, Any]) -> pd.DataFrame:
+ """Generate a dataset from evaluation data.
+
+ Args:
+ data (List[Dict[str, Any]]): The data to generate a dataset for.
+ config (Dict[str, Any]): The configuration of the task.
+
+ Returns:
+ pd.DataFrame: A dataframe that is ready to be uploaded to W&B.
+ """
+ ids = [x["doc_id"] for x in data]
+ labels = [x["target"] for x in data]
+ instance = [""] * len(ids)
+ resps = [""] * len(ids)
+ filtered_resps = [""] * len(ids)
+ model_outputs = {}
+
+ metrics_list = config["metric_list"]
+ metrics = {}
+ for metric in metrics_list:
+ metric = metric.get("metric")
+ if metric in ["word_perplexity", "byte_perplexity", "bits_per_byte"]:
+ metrics[f"{metric}_loglikelihood"] = [x[metric][0] for x in data]
+ if metric in ["byte_perplexity", "bits_per_byte"]:
+ metrics[f"{metric}_bytes"] = [x[metric][1] for x in data]
+ else:
+ metrics[f"{metric}_words"] = [x[metric][1] for x in data]
+ else:
+ metrics[metric] = [x[metric] for x in data]
+
+ if config["output_type"] == "loglikelihood":
+ instance = [x["arguments"][0][0] for x in data]
+ labels = [x["arguments"][0][1] for x in data]
+ resps = [f'log probability of continuation is {x["resps"][0][0][0]} ' + "\n\n" + "continuation will {} generated with greedy sampling".format("not be" if not x["resps"][0][0][1] else "be") for x in data]
+ filtered_resps = [f'log probability of continuation is {x["filtered_resps"][0][0]} ' + "\n\n" + "continuation will {} generated with greedy sampling".format("not be" if not x["filtered_resps"][0][1] else "be") for x in data]
+ elif config["output_type"] == "multiple_choice":
+ instance = [x["arguments"][0][0] for x in data]
+ choices = ["\n".join([f"{idx}. {y[1]}" for idx, y in enumerate(x["arguments"])]) for x in data]
+ resps = [np.argmax([n[0][0] for n in x["resps"]]) for x in data]
+ filtered_resps = [np.argmax([n[0] for n in x["filtered_resps"]]) for x in data]
+ elif config["output_type"] == "loglikelihood_rolling":
+ instance = [x["arguments"][0][0] for x in data]
+ resps = [x["resps"][0][0] for x in data]
+ filtered_resps = [x["filtered_resps"][0] for x in data]
+ elif config["output_type"] == "generate_until":
+ instance = [x["arguments"][0][0] for x in data]
+ resps = [x["resps"][0][0] for x in data]
+ filtered_resps = [x["filtered_resps"][0] for x in data]
+
+ model_outputs["raw_predictions"] = resps
+ model_outputs["filtered_predictions"] = filtered_resps
+
+ df_data = {
+ "id": ids,
+ "data": instance,
+ }
+ if config["output_type"] == "multiple_choice":
+ df_data["choices"] = choices
+
+ tmp_data = {
+ "input_len": [len(x) for x in instance],
+ "labels": labels,
+ "output_type": config["output_type"],
+ }
+ df_data.update(tmp_data)
+ df_data.update(model_outputs)
+ df_data.update(metrics)
+
+ return pd.DataFrame(df_data)
+
+ def _log_samples_as_artifact(self, data: List[Dict[str, Any]], task_name: str) -> None:
+ import wandb
+
+ # log the samples as an artifact
+ dumped = json.dumps(
+ data,
+ indent=2,
+ default=_handle_non_serializable,
+ ensure_ascii=False,
+ )
+ artifact = wandb.Artifact(f"{task_name}", type="samples_by_task")
+ with artifact.new_file(f"{task_name}_eval_samples.json", mode="w", encoding="utf-8") as f:
+ f.write(dumped)
+ self.run.log_artifact(artifact)
+ # artifact.wait()
+
+ def log_eval_samples(self, samples: Dict[str, List[Dict[str, Any]]]) -> None:
+ """Log evaluation samples to W&B.
+
+ Args:
+ samples (Dict[str, List[Dict[str, Any]]]): Evaluation samples for each task.
+ """
+ task_names: List[str] = [x for x in self.task_names if x not in self.group_names]
+
+ ungrouped_tasks = []
+ tasks_by_groups = {}
+
+ for task_name in task_names:
+ group_names = self.task_configs[task_name].get("group", None)
+ if group_names:
+ if isinstance(group_names, str):
+ group_names = [group_names]
+
+ for group_name in group_names:
+ if not tasks_by_groups.get(group_name):
+ tasks_by_groups[group_name] = [task_name]
+ else:
+ tasks_by_groups[group_name].append(task_name)
+ else:
+ ungrouped_tasks.append(task_name)
+
+ for task_name in ungrouped_tasks:
+ eval_preds = samples[task_name]
+
+ # log the samples as a W&B Table
+ df = self._generate_dataset(eval_preds, self.task_configs.get(task_name))
+ self.run.log({f"{task_name}_eval_results": df})
+
+ # log the samples as a json file as W&B Artifact
+ self._log_samples_as_artifact(eval_preds, task_name)
+
+ for group, grouped_tasks in tasks_by_groups.items():
+ grouped_df = pd.DataFrame()
+ for task_name in grouped_tasks:
+ eval_preds = samples[task_name]
+ df = self._generate_dataset(eval_preds, self.task_configs.get(task_name))
+ df["group"] = group
+ df["task"] = task_name
+ grouped_df = pd.concat([grouped_df, df], ignore_index=True)
+
+ # log the samples as a json file as W&B Artifact
+ self._log_samples_as_artifact(eval_preds, task_name)
+
+ self.run.log({f"{group}_eval_results": grouped_df})
diff --git a/lmms-eval/lmms_eval/logging_utils.py b/lmms-eval/lmms_eval/logging_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..27f25513200903702b9a122737d622e60ef9d071
--- /dev/null
+++ b/lmms-eval/lmms_eval/logging_utils.py
@@ -0,0 +1,368 @@
+# Code mostly from: https://github.com/EleutherAI/lm-evaluation-harness/pull/1339, credit to: https://github.com/ayulockin
+import copy
+import glob
+import json
+import os
+import re
+from datetime import datetime
+from typing import Any, Dict, List, Literal, Tuple, Union
+
+import numpy as np
+import pandas as pd
+import tenacity
+from loguru import logger
+from packaging.version import Version
+
+from lmms_eval import utils
+
+try:
+ import wandb
+
+ assert Version(wandb.__version__) >= Version("0.13.6")
+ if Version(wandb.__version__) < Version("0.13.6"):
+ wandb.require("report-editing:v0")
+except Exception as e:
+ logger.warning("To use the wandb reporting functionality please install wandb>=0.13.6.\n" "To install the latest version of wandb run `pip install wandb --upgrade`\n" f"{e}")
+
+
+def remove_none_pattern(input_string):
+ # Define the pattern to match ',none' at the end of the string
+ pattern = re.compile(r",none$")
+
+ # Use sub() to replace ',none' with an empty string
+ result = re.sub(pattern, "", input_string)
+
+ # check if the input_string changed
+ removed = result != input_string
+
+ return result, removed
+
+
+def _handle_non_serializable(o: Any) -> Union[int, str, list]:
+ """Handle non-serializable objects by converting them to serializable types.
+
+ Args:
+ o (Any): The object to be handled.
+
+ Returns:
+ Union[int, str, list]: The converted object. If the object is of type np.int64 or np.int32,
+ it will be converted to int. If the object is of type set, it will be converted
+ to a list. Otherwise, it will be converted to str.
+ """
+ if isinstance(o, np.int64) or isinstance(o, np.int32):
+ return int(o)
+ elif isinstance(o, set):
+ return list(o)
+ else:
+ return str(o)
+
+
+def get_wandb_printer() -> Literal["Printer"]:
+ """Returns a wandb printer instance for pretty stdout."""
+ from wandb.sdk.lib.printer import get_printer
+ from wandb.sdk.wandb_settings import Settings
+
+ printer = get_printer(Settings()._jupyter)
+ return printer
+
+
+# class WandbLogger:
+class WandbLogger:
+ def __init__(self, args):
+ self.wandb_args = utils.simple_parse_args_string(args.wandb_args)
+ self.args = args
+ self.all_args_dict = vars(args)
+ self.printer = get_wandb_printer()
+ try:
+ self.init_run()
+ except Exception as e:
+ logger.warning(f"Failed to initialize W&B run: {e}")
+ os.environ["WANDB_MODE"] = "offline"
+ self.init_run()
+
+ def finish(self):
+ self.run.finish()
+
+ @tenacity.retry(wait=tenacity.wait_fixed(5), stop=tenacity.stop_after_attempt(5))
+ def init_run(self):
+ if "name" not in self.wandb_args:
+ if "config" in self.all_args_dict and self.all_args_dict["config"] != "":
+ self.wandb_args["name"] = self.all_args_dict["config"].split("/")[-1].replace(".yaml", "") + "/" + self.args.log_samples_suffix
+ else:
+ task_names = self.args.tasks.replace(",", "/")
+ self.wandb_args["name"] = f"{self.args.model}/<{task_names}>/{self.args.log_samples_suffix}"
+ if self.args.num_fewshot:
+ self.wandb_args["name"] += f"_{self.args.num_fewshot}shot"
+ if "project" not in self.wandb_args:
+ self.wandb_args["project"] = "lmms-eval"
+ # initialize a W&B run
+ self.run = wandb.init(**self.wandb_args)
+
+ def post_init(self, results: Dict[str, Any]) -> None:
+ self.results: Dict[str, Any] = copy.deepcopy(results)
+ self.task_names: List[str] = list(results.get("results", {}).keys())
+ self.group_names: List[str] = list(results.get("groups", {}).keys())
+
+ def _get_config(self) -> Dict[str, Any]:
+ """Get configuration parameters."""
+ self.task_configs = self.results.get("configs", {})
+ cli_configs = self.results.get("config", {})
+ configs = {
+ "task_configs": self.task_configs,
+ "cli_configs": cli_configs,
+ }
+
+ return configs
+
+ def _sanitize_results_dict(self) -> Tuple[Dict[str, str], Dict[str, Any]]:
+ """Sanitize the results dictionary."""
+ _results = copy.deepcopy(self.results.get("results", dict()))
+ _results["model_configs"] = self.results.get("model_configs", dict())
+
+ # Remove None from the metric string name
+ tmp_results = copy.deepcopy(_results)
+ for task_name in self.task_names:
+ task_result = tmp_results.get(task_name, dict())
+ for metric_name, metric_value in task_result.items():
+ _metric_name, removed = remove_none_pattern(metric_name)
+ if removed:
+ _results[task_name][_metric_name] = metric_value
+ _results[task_name].pop(metric_name)
+
+ # remove string valued keys from the results dict
+ wandb_summary = {}
+ for task in self.task_names:
+ task_result = _results.get(task, dict())
+ for metric_name, metric_value in task_result.items():
+ if isinstance(metric_value, str):
+ wandb_summary[f"{task}/{metric_name}"] = metric_value
+
+ wandb_summary["model_configs"] = self.results.get("model_configs", dict())
+ for summary_metric, summary_value in wandb_summary.items():
+ if summary_metric != "model_configs":
+ _task, _summary_metric = summary_metric.split("/")
+ _results[_task].pop(_summary_metric)
+
+ tmp_results = copy.deepcopy(_results)
+ for task_name, task_results in tmp_results.items():
+ if task_name != "model_configs":
+ for metric_name, metric_value in task_results.items():
+ _results[f"{task_name}/{metric_name}"] = metric_value
+ _results[task_name].pop(metric_name)
+ for task in self.task_names:
+ _results.pop(task)
+
+ return wandb_summary, _results
+
+ def _log_results_as_table(self) -> None:
+ """Generate and log evaluation results as a table to W&B."""
+ columns = [
+ "Model",
+ "Args",
+ "Tasks",
+ "Version",
+ "Filter",
+ "num_fewshot",
+ "Metric",
+ "Value",
+ "Stderr",
+ ]
+
+ def make_table(columns: List[str], key: str = "results"):
+ table = wandb.Table(columns=columns)
+ results = copy.deepcopy(self.results)
+
+ model_name = results.get("model_configs").get("model")
+ model_args = results.get("model_configs").get("model_args")
+
+ for k, dic in results.get(key).items():
+ if k in self.group_names and not key == "groups":
+ continue
+ version = results.get("versions").get(k)
+ if version == "N/A":
+ version = None
+ n = results.get("n-shot").get(k)
+
+ for (mf), v in dic.items():
+ m, _, f = mf.partition(",")
+ if m.endswith("_stderr"):
+ continue
+ if m == "alias":
+ continue
+
+ if m + "_stderr" + "," + f in dic:
+ se = dic[m + "_stderr" + "," + f]
+ if se != "N/A":
+ se = "%.4f" % se
+ data = [model_name, model_args, k, version, f, n, m, str(v), str(se)]
+ if key == "groups":
+ data = [self.group_names] + data
+ table.add_data(*data)
+ else:
+ data = [model_name, model_args, k, version, f, n, m, str(v), ""]
+ if key == "groups":
+ data = [self.group_names] + data
+ table.add_data(*data)
+
+ return table
+
+ # log the complete eval result to W&B Table
+ table = make_table(columns, "results")
+ self.run.log({"evaluation/eval_results": table})
+
+ if "groups" in self.results.keys():
+ table = make_table(["Groups"] + columns, "groups")
+ self.run.log({"evaluation/group_eval_results": table})
+
+ def _log_results_as_artifact(self) -> None:
+ """Log results as JSON artifact to W&B."""
+ dumped = json.dumps(self.results, indent=2, default=_handle_non_serializable, ensure_ascii=False)
+ artifact = wandb.Artifact("results", type="eval_results")
+ with artifact.new_file("results.json", mode="w", encoding="utf-8") as f:
+ f.write(dumped)
+ self.run.log_artifact(artifact)
+
+ def log_eval_result(self) -> None:
+ """Log evaluation results to W&B."""
+ # Log configs to wandb
+ configs = self._get_config()
+ self.run.config.update(configs, allow_val_change=True)
+
+ wandb_summary, self.wandb_results = self._sanitize_results_dict()
+ # update wandb.run.summary with items that were removed
+ self.run.summary.update(wandb_summary)
+ # Log the evaluation metrics to wandb
+ self.run.log(self.wandb_results)
+ # Log the evaluation metrics as W&B Table
+ self._log_results_as_table()
+ # Log the results dict as json to W&B Artifacts
+ self._log_results_as_artifact()
+
+ def _generate_dataset(self, data: List[Dict[str, Any]], config: Dict[str, Any]) -> pd.DataFrame:
+ """Generate a dataset from evaluation data.
+
+ Args:
+ data (List[Dict[str, Any]]): The data to generate a dataset for.
+ config (Dict[str, Any]): The configuration of the task.
+
+ Returns:
+ pd.DataFrame: A dataframe that is ready to be uploaded to W&B.
+ """
+ ids = [x["doc_id"] for x in data]
+ labels = [x["target"] for x in data]
+ instance = [""] * len(ids)
+ resps = [""] * len(ids)
+ filtered_resps = [""] * len(ids)
+ model_outputs = {}
+
+ metrics_list = config["metric_list"]
+ metrics = {}
+ for metric in metrics_list:
+ metric = metric.get("metric")
+ if metric in ["word_perplexity", "byte_perplexity", "bits_per_byte"]:
+ metrics[f"{metric}_loglikelihood"] = [x[metric][0] for x in data]
+ if metric in ["byte_perplexity", "bits_per_byte"]:
+ metrics[f"{metric}_bytes"] = [x[metric][1] for x in data]
+ else:
+ metrics[f"{metric}_words"] = [x[metric][1] for x in data]
+ else:
+ metrics[metric] = [x[metric] for x in data]
+
+ if config["output_type"] == "loglikelihood":
+ instance = [x["arguments"][0][0] for x in data]
+ labels = [x["arguments"][0][1] for x in data]
+ resps = [f'log probability of continuation is {x["resps"][0][0][0]} ' + "\n\n" + "continuation will {} generated with greedy sampling".format("not be" if not x["resps"][0][0][1] else "be") for x in data]
+ filtered_resps = [f'log probability of continuation is {x["filtered_resps"][0][0]} ' + "\n\n" + "continuation will {} generated with greedy sampling".format("not be" if not x["filtered_resps"][0][1] else "be") for x in data]
+ elif config["output_type"] == "multiple_choice":
+ instance = [x["arguments"][0][0] for x in data]
+ choices = ["\n".join([f"{idx}. {y[1]}" for idx, y in enumerate(x["arguments"])]) for x in data]
+ resps = [np.argmax([n[0][0] for n in x["resps"]]) for x in data]
+ filtered_resps = [np.argmax([n[0] for n in x["filtered_resps"]]) for x in data]
+ elif "generate_until" in config["output_type"]:
+ instance = [x["arguments"][0][0] for x in data]
+ resps = [x["resps"][0][0] for x in data]
+ filtered_resps = [x["filtered_resps"][0] for x in data]
+
+ model_outputs["raw_predictions"] = resps
+ model_outputs["filtered_predictions"] = filtered_resps
+
+ df_data = {
+ "id": ids,
+ "data": instance,
+ }
+ if config["output_type"] == "multiple_choice":
+ df_data["choices"] = choices
+
+ tmp_data = {
+ "input_len": [len(x) for x in instance],
+ "labels": labels,
+ "output_type": config["output_type"],
+ }
+ df_data.update(tmp_data)
+ df_data.update(model_outputs)
+ df_data.update(metrics)
+
+ return pd.DataFrame(df_data)
+
+ def _log_samples_as_artifact(self, data: List[Dict[str, Any]], task_name: str) -> None:
+ # log the samples as an artifact
+ dumped = json.dumps(
+ data,
+ indent=2,
+ default=_handle_non_serializable,
+ ensure_ascii=False,
+ )
+ artifact = wandb.Artifact(f"{task_name}", type="samples_by_task")
+ with artifact.new_file(f"{task_name}_eval_samples.json", mode="w", encoding="utf-8") as f:
+ f.write(dumped)
+ self.run.log_artifact(artifact)
+ # artifact.wait()
+
+ def log_eval_samples(self, samples: Dict[str, List[Dict[str, Any]]]) -> None:
+ """Log evaluation samples to W&B.
+
+ Args:
+ samples (Dict[str, List[Dict[str, Any]]]): Evaluation samples for each task.
+ """
+ task_names: List[str] = [x for x in self.task_names if x not in self.group_names]
+
+ ungrouped_tasks = []
+ tasks_by_groups = {}
+
+ for task_name in task_names:
+ group_names = self.task_configs[task_name].get("group", None)
+ if group_names:
+ if isinstance(group_names, str):
+ group_names = [group_names]
+
+ for group_name in group_names:
+ if not tasks_by_groups.get(group_name):
+ tasks_by_groups[group_name] = [task_name]
+ else:
+ tasks_by_groups[group_name].append(task_name)
+ else:
+ ungrouped_tasks.append(task_name)
+
+ for task_name in ungrouped_tasks:
+ eval_preds = samples[task_name]
+
+ # log the samples as a W&B Table
+ df = self._generate_dataset(eval_preds, self.task_configs.get(task_name))
+ self.run.log({f"{task_name}_eval_results": df})
+
+ # log the samples as a json file as W&B Artifact
+ self._log_samples_as_artifact(eval_preds, task_name)
+
+ for group, grouped_tasks in tasks_by_groups.items():
+ grouped_df = pd.DataFrame()
+ for task_name in grouped_tasks:
+ eval_preds = samples[task_name]
+ df = self._generate_dataset(eval_preds, self.task_configs.get(task_name))
+ df["group"] = group
+ df["task"] = task_name
+ grouped_df = pd.concat([grouped_df, df], ignore_index=True)
+
+ # log the samples as a json file as W&B Artifact
+ self._log_samples_as_artifact(eval_preds, task_name)
+
+ self.run.log({f"{group}_eval_results": grouped_df})
diff --git a/lmms-eval/lmms_eval/mcp/__init__.py b/lmms-eval/lmms_eval/mcp/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..aea27316b151244cc9a0c09d4ecfadd2a785f6c5
--- /dev/null
+++ b/lmms-eval/lmms_eval/mcp/__init__.py
@@ -0,0 +1,3 @@
+from .client import MCPClient
+
+__all__ = ["MCPClient"]
diff --git a/lmms-eval/lmms_eval/models/__init__.py b/lmms-eval/lmms_eval/models/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..65be32a53b0924843f7e48b60a1d809c2d2c9ed1
--- /dev/null
+++ b/lmms-eval/lmms_eval/models/__init__.py
@@ -0,0 +1,119 @@
+import importlib
+import os
+import sys
+from typing import Literal
+
+from loguru import logger
+
+# os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
+
+logger.remove()
+# Configure logger with detailed format including file path, function name, and line number
+log_format = "{time:YYYY-MM-DD HH:mm:ss} | " "{level: <8} | " "{name}:{function}:{line} - " "{message}"
+logger.add(sys.stdout, level="WARNING", format=log_format)
+
+
+AVAILABLE_SIMPLE_MODELS = {
+ "aero": "Aero",
+ "plm": "PerceptionLM",
+ "aria": "Aria",
+ "auroracap": "AuroraCap",
+ "batch_gpt4": "BatchGPT4",
+ "claude": "Claude",
+ "cogvlm2": "CogVLM2",
+ "from_log": "FromLog",
+ "fuyu": "Fuyu",
+ "gemini_api": "GeminiAPI",
+ "gpt4v": "GPT4V",
+ "idefics2": "Idefics2",
+ "instructblip": "InstructBLIP",
+ "internvideo2": "InternVideo2",
+ "internvl": "InternVLChat",
+ "internvl2": "InternVL2",
+ "llama_vid": "LLaMAVid",
+ "llama_vision": "LlamaVision",
+ "llava": "Llava",
+ "llava_hf": "LlavaHf",
+ "llava_onevision": "Llava_OneVision",
+ "llava_onevision_moviechat": "Llava_OneVision_MovieChat",
+ "llava_sglang": "LlavaSglang",
+ "llava_vid": "LlavaVid",
+ "longva": "LongVA",
+ "mantis": "Mantis",
+ "minicpm_v": "MiniCPM_V",
+ "minimonkey": "MiniMonkey",
+ "moviechat": "MovieChat",
+ "mplug_owl_video": "mplug_Owl",
+ "ola": "Ola",
+ "openai_compatible": "OpenAICompatible",
+ "oryx": "Oryx",
+ "phi3v": "Phi3v",
+ "phi4_multimodal": "Phi4",
+ "qwen2_5_omni": "Qwen2_5_Omni",
+ "qwen2_5_vl": "Qwen2_5_VL",
+ "qwen2_5_vl_interleave": "Qwen2_5_VL_Interleave",
+ "qwen2_audio": "Qwen2_Audio",
+ "qwen2_vl": "Qwen2_VL",
+ "qwen_vl": "Qwen_VL",
+ "qwen_vl_api": "Qwen_VL_API",
+ "reka": "Reka",
+ "ross": "Ross",
+ "slime": "Slime",
+ "srt_api": "SRT_API",
+ "tinyllava": "TinyLlava",
+ "videoChatGPT": "VideoChatGPT",
+ "videochat2": "VideoChat2",
+ "videollama3": "VideoLLaMA3",
+ "video_llava": "VideoLLaVA",
+ "vila": "VILA",
+ "vita": "VITA",
+ "vllm": "VLLM",
+ "xcomposer2_4KHD": "XComposer2_4KHD",
+ "xcomposer2d5": "XComposer2D5",
+ "egogpt": "EgoGPT",
+ "internvideo2_5": "InternVideo2_5",
+ "videochat_flash": "VideoChat_Flash",
+ "whisper": "Whisper",
+ "whisper_vllm": "WhisperVllm",
+ "vora": "VoRA",
+}
+
+AVAILABLE_CHAT_TEMPLATE_MODELS = {"llava_hf": "LlavaHf", "qwen2_5_vl": "Qwen2_5_VL", "openai_compatible": "OpenAICompatible", "vllm": "VLLM", "sglang": "Sglang", "huggingface": "Huggingface", "async_openai": "AsyncOpenAIChat"}
+
+
+def get_model(model_name, force_simple: bool = False):
+ if model_name not in AVAILABLE_SIMPLE_MODELS and model_name not in AVAILABLE_CHAT_TEMPLATE_MODELS:
+ raise ValueError(f"Model {model_name} not found in available models.")
+
+ if model_name in AVAILABLE_CHAT_TEMPLATE_MODELS:
+ model_type = "chat"
+ AVAILABLE_MODELS = AVAILABLE_CHAT_TEMPLATE_MODELS
+ else:
+ model_type = "simple"
+ AVAILABLE_MODELS = AVAILABLE_SIMPLE_MODELS
+
+ # Override with force_simple if needed, but only if the model exists in AVAILABLE_SIMPLE_MODELS
+ if force_simple and model_name in AVAILABLE_SIMPLE_MODELS:
+ model_type = "simple"
+ AVAILABLE_MODELS = AVAILABLE_SIMPLE_MODELS
+
+ model_class = AVAILABLE_MODELS[model_name]
+ if "." not in model_class:
+ model_class = f"lmms_eval.models.{model_type}.{model_name}.{model_class}"
+
+ try:
+ model_module, model_class = model_class.rsplit(".", 1)
+ module = __import__(model_module, fromlist=[model_class])
+ return getattr(module, model_class)
+ except Exception as e:
+ logger.error(f"Failed to import {model_class} from {model_name}: {e}")
+ raise
+
+
+if os.environ.get("LMMS_EVAL_PLUGINS", None):
+ # Allow specifying other packages to import models from
+ for plugin in os.environ["LMMS_EVAL_PLUGINS"].split(","):
+ m = importlib.import_module(f"{plugin}.models")
+ # For plugin users, this will be replaced by chat template model later
+ for model_name, model_class in getattr(m, "AVAILABLE_MODELS").items():
+ AVAILABLE_SIMPLE_MODELS[model_name] = f"{plugin}.models.{model_name}.{model_class}"
diff --git a/lmms-eval/lmms_eval/models/chat/async_openai.py b/lmms-eval/lmms_eval/models/chat/async_openai.py
new file mode 100644
index 0000000000000000000000000000000000000000..7c810e81a38872327be6a32b920958bfcf8f191d
--- /dev/null
+++ b/lmms-eval/lmms_eval/models/chat/async_openai.py
@@ -0,0 +1,179 @@
+import asyncio
+import base64
+import json
+import os
+import time
+import uuid
+from io import BytesIO
+from multiprocessing import cpu_count
+from typing import List, Tuple, Union
+
+import numpy as np
+import requests as url_requests
+from accelerate import Accelerator, DistributedType
+from tqdm import tqdm
+
+from lmms_eval.api.instance import Instance
+from lmms_eval.api.model import lmms
+from lmms_eval.api.registry import register_model
+
+try:
+ from decord import VideoReader, cpu
+except ImportError:
+ pass
+
+from dotenv import find_dotenv, load_dotenv
+from loguru import logger as eval_logger
+from openai import AsyncOpenAI
+from PIL import Image
+
+from lmms_eval.api.model import lmms
+from lmms_eval.mcp import MCPClient
+from lmms_eval.models.simple.openai_compatible import (
+ OpenAICompatible as OpenAICompatibleSimple,
+)
+from lmms_eval.protocol import ChatMessages
+
+load_dotenv(verbose=True)
+
+
+@register_model("async_openai_compatible_chat")
+class AsyncOpenAIChat(lmms):
+ is_simple = False
+
+ def __init__(
+ self,
+ model_version: str = "grok-2-latest",
+ timeout: int = 10,
+ max_retries: int = 5,
+ max_size_in_mb: int = 20,
+ mcp_server_path: str = None,
+ num_cpus: int = None,
+ work_dir: str = "./workspace",
+ **kwargs,
+ ) -> None:
+ super().__init__()
+ self.model_version = model_version
+ self.timeout = timeout
+ self.max_retries = max_retries
+ self.max_size_in_mb = max_size_in_mb # some models have a limit on the size of the image
+ if num_cpus is None:
+ self.num_cpus = cpu_count() // 2
+ else:
+ self.num_cpus = num_cpus
+ self.work_dir = work_dir
+
+ self.client = AsyncOpenAI(api_key=os.getenv("OPENAI_API_KEY"), base_url=os.getenv("OPENAI_API_BASE"), timeout=timeout)
+ if mcp_server_path is not None:
+ self.mcp_client = MCPClient(mcp_server_path)
+ os.makedirs(self.work_dir, exist_ok=True)
+ else:
+ self.mcp_client = None
+
+ @property
+ def model(self):
+ # returns the model, unwrapping it if using Accelerate
+ return self.client
+
+ @property
+ def batch_size(self):
+ return self.batch_size_per_gpu
+
+ @property
+ def rank(self):
+ return self._rank
+
+ @property
+ def world_size(self):
+ return self._world_size
+
+ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
+ assert False, "TODO, not implemented"
+
+ def generate_until_multi_round(self, requests) -> List[str]:
+ raise NotImplementedError("TODO: Implement multi-round generation for LLaVAHF")
+
+ async def maybe_forward_with_tool(self, request: Instance, idx: int):
+ """
+ Forward the request to the OpenAI API, using tools if available.
+ This method is designed to handle chat messages and tool calls in an asynchronous manner.
+ It retrieves the chat messages from the request, prepares the payload, and sends it to the OpenAI API.
+ If the response indicates that tool calls are needed, it will call the tools using the MCP client and continue the conversation until a final response is received.
+ :param request: The request instance containing the chat messages and other parameters.
+ :param idx: The index of the request in the batch. (Use to restore the original order of responses)
+ """
+ ctx, doc_to_messages, gen_kwargs, doc_id, task, split = request.args
+ chat_messages = doc_to_messages(self.task_dict[task][split][doc_id])
+ chat_messages: ChatMessages = ChatMessages(**{"messages": chat_messages})
+ messages = chat_messages.to_openai_messages()
+ images, videos, audios = chat_messages.extract_media()
+ if self.mcp_client is not None:
+ for image in images:
+ image_path = os.path.join(self.work_dir, f"{uuid.uuid4()}.jpg")
+ image.save(image_path)
+ messages[-1]["content"].append({"type": "text", "text": f"Image 1 has image path: {image_path}"})
+
+ payload = {"messages": messages}
+ payload["model"] = self.model_version
+
+ if "max_new_tokens" not in gen_kwargs:
+ gen_kwargs["max_new_tokens"] = 1024
+ if "temperature" not in gen_kwargs:
+ gen_kwargs["temperature"] = 0
+ if "top_p" not in gen_kwargs:
+ gen_kwargs["top_p"] = None
+ # payload["max_completion_tokens"] = gen_kwargs["max_new_tokens"]
+ payload["max_tokens"] = gen_kwargs["max_new_tokens"]
+ payload["temperature"] = gen_kwargs["temperature"]
+
+ if self.mcp_client is not None:
+ # get the function list from the MCP server
+ functions = await self.mcp_client.get_function_list()
+ payload["tools"] = functions
+ payload["tool_choice"] = "auto" # or "auto" for automatic tool selection
+
+ response = await self.client.chat.completions.create(**payload)
+
+ while response.choices[0].finish_reason == "tool_calls":
+ message = response.choices[0].message
+ tool_messages = []
+ if message.tool_calls:
+ eval_logger.debug("Calling tool with MCP client")
+ for call in message.tool_calls:
+ eval_logger.debug(f"Calling {call.function.name}...")
+ result = await self.mcp_client.run_tool(call.function.name, eval(call.function.arguments))
+ for content in result.content:
+ tool_message = self.mcp_client.convert_result_to_openai_format(result)
+ tool_messages.append({"role": "tool", "name": call.function.name, "content": tool_message})
+
+ response = await self.client.chat.completions.create(
+ model=self.model_version,
+ messages=message + tool_messages,
+ max_tokens=gen_kwargs["max_new_tokens"],
+ temperature=gen_kwargs["temperature"],
+ )
+
+ return response.choices[0].message.content, idx
+
+ def generate_until(self, requests) -> List[str]:
+ async def run():
+ res = []
+ pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
+ sem = asyncio.Semaphore(self.num_cpus)
+
+ async def _process(req, idx):
+ async with sem:
+ return await self.maybe_forward_with_tool(req, idx)
+
+ tasks = [asyncio.create_task(_process(req, idx)) for idx, req in enumerate(requests)]
+ for task in asyncio.as_completed(tasks):
+ content, idx = await task
+ res.append((content, idx))
+ pbar.update(1)
+
+ pbar.close()
+ return res
+
+ results = asyncio.run(run())
+ results.sort(key=lambda x: x[1]) # Sort by index to restore original
+ return [content for content, _ in results]
diff --git a/lmms-eval/lmms_eval/models/chat/llava_hf.py b/lmms-eval/lmms_eval/models/chat/llava_hf.py
new file mode 100644
index 0000000000000000000000000000000000000000..480caf81e124ec71ce5f11e48273e331dc1ca26e
--- /dev/null
+++ b/lmms-eval/lmms_eval/models/chat/llava_hf.py
@@ -0,0 +1,136 @@
+import time
+import warnings
+from typing import List, Optional, Tuple, Union
+
+from tqdm import tqdm
+
+from lmms_eval import utils
+from lmms_eval.api.instance import Instance
+from lmms_eval.api.model import lmms
+from lmms_eval.api.registry import register_model
+from lmms_eval.models.model_utils.gen_metrics import log_metrics
+from lmms_eval.protocol import ChatMessages
+
+warnings.filterwarnings("ignore")
+
+from loguru import logger as eval_logger
+
+from lmms_eval.api.registry import register_model
+from lmms_eval.models.simple.llava_hf import LlavaHf as LlavaHfSimple
+
+DEFAULT_IMAGE_TOKEN = ""
+DEFAULT_VIDEO_TOKEN = "