Add files using upload-large-folder tool

lm-evaluation-harness/tests/testdata/wmt20-en-zh-v0-greedy_until

@@ -0,0 +1 @@
+67f0333ddbcb07d7a9ac12919129a18fe4fea24e4826a11bbdde4fd5ed5ed83f

lm-evaluation-harness/tests/testdata/wmt20-en-zh-v0-res.json

@@ -0,0 +1 @@
+{"results": {"wmt20-en-zh": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.00014170297316825535, "chrf_stderr": 6.590669847391838e-05, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-en-zh": 0}}

lm-evaluation-harness/tests/testdata/wmt20-fr-de-v0-res.json

@@ -0,0 +1 @@
+{"results": {"wmt20-fr-de": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.01143193767396364, "chrf_stderr": 0.00012555271954563658, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-fr-de": 0}}

lm-evaluation-harness/tests/testdata/wmt20-iu-en-v0-greedy_until

@@ -0,0 +1 @@
+97bf664a8efa54b5366b8341f77b418106dd0cb26169d5b2d0144e4d3d2bc5c9

lm-evaluation-harness/tests/testdata/wmt20-iu-en-v0-res.json

@@ -0,0 +1 @@
+{"results": {"wmt20-iu-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.012204628007572778, "chrf_stderr": 8.944407532175802e-05, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-iu-en": 0}}

lm-evaluation-harness/tests/testdata/wmt20-ja-en-v0-greedy_until

@@ -0,0 +1 @@
+1fd846f3c0104e794eb380dae7f648592092ab8bf59234c26d0a671bbbc28df1

lm-evaluation-harness/tests/testdata/wmt20-ja-en-v0-res.json

@@ -0,0 +1 @@
+{"results": {"wmt20-ja-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.010703148854351403, "chrf_stderr": 0.00022242113108130186, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-ja-en": 0}}

lm-evaluation-harness/tests/testdata/wmt20-km-en-v0-greedy_until

@@ -0,0 +1 @@
+fb4ec81bb89c70df7e21b43e0e882915b7b71a2a85bb8d4b59e0c7938baaa4c2

lm-evaluation-harness/tests/testdata/wmt20-km-en-v0-res.json

@@ -0,0 +1 @@
+{"results": {"wmt20-km-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.015142474534585969, "chrf_stderr": 0.0001518735048829897, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-km-en": 0}}

lm-evaluation-harness/tests/testdata/wmt20-pl-en-v0-greedy_until

@@ -0,0 +1 @@
+89274499d84176b1ffe4eaec06f2c89ca807342384dc946c2e348d00116aaade

lm-evaluation-harness/tests/testdata/wmt20-pl-en-v0-res.json

@@ -0,0 +1 @@
+{"results": {"wmt20-pl-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.01353367757716276, "chrf_stderr": 0.00018386199249976465, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-pl-en": 0}}

lm-evaluation-harness/tests/testdata/wmt20-ps-en-v0-greedy_until

@@ -0,0 +1 @@
+c3976465e3709b4bc371175cc1494c69fe096ea4ba7d114da779d2baa0a47466

lm-evaluation-harness/tests/testdata/wmt20-ps-en-v0-res.json

@@ -0,0 +1 @@
+{"results": {"wmt20-ps-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.015192865365105723, "chrf_stderr": 0.00011334541381539086, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-ps-en": 0}}

lm-evaluation-harness/tests/testdata/wmt20-ru-en-v0-greedy_until

@@ -0,0 +1 @@
+1477ab6542c26bd0222cc1aded174f33bf8d04d1cf6a1c0959aeca4ff3779adc

lm-evaluation-harness/tests/testdata/wmt20-ru-en-v0-res.json

@@ -0,0 +1 @@
+{"results": {"wmt20-ru-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.013344639906399232, "chrf_stderr": 7.583552652374546e-05, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-ru-en": 0}}

lm-evaluation-harness/tests/testdata/wmt20-ta-en-v0-greedy_until

@@ -0,0 +1 @@
+111ea3efdc08f1cf536631b9426c3a20e482c575d009d2a8c71f59c027578eec

lm-evaluation-harness/tests/testdata/wmt20-ta-en-v0-res.json

@@ -0,0 +1 @@
+{"results": {"wmt20-ta-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.013841110664859798, "chrf_stderr": 0.00018476696850880766, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-ta-en": 0}}

lm-evaluation-harness/tests/testdata/wmt20-zh-en-v0-greedy_until

@@ -0,0 +1 @@
+07dbadfd6f2b2b9462ab6187dbfaabae6e5192ab89a8e4ede9237834b9364dd1

lm-evaluation-harness/tests/testdata/wmt20-zh-en-v0-res.json

@@ -0,0 +1 @@
+{"results": {"wmt20-zh-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.008438201290981157, "chrf_stderr": 0.0001109053964076822, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-zh-en": 0}}

lm-evaluation-harness/tests/testdata/wnli-v0-loglikelihood

@@ -0,0 +1 @@
+2ffd304d6096416eb29607e2e7642b1d6043163624967bcf4c4fc00fddc6c721

lm-evaluation-harness/tests/testdata/wnli-v0-res.json

@@ -0,0 +1 @@
+{"results": {"wnli": {"acc": 0.3380281690140845, "acc_stderr": 0.05653887739133514}}, "versions": {"wnli": 0}}

lm-evaluation-harness/tests/testdata/wnli-v1-loglikelihood

@@ -0,0 +1 @@
+8a0f81661d2ab2334bbc8031fac31c0c8882f1d9271dd51599d21dfdbb726dea

lm-evaluation-harness/tests/testdata/wnli-v1-res.json

@@ -0,0 +1 @@
+{"results": {"wnli": {"acc": 0.5633802816901409, "acc_stderr": 0.0592793555841297}}, "versions": {"wnli": 1}}

lm-evaluation-harness/tests/testdata/wsc-v0-loglikelihood

@@ -0,0 +1 @@
+45865468eff5ca31e6a050947a6b3310d9d5ed19d0f2e578a32ecaf1c768600f

lm-evaluation-harness/tests/testdata/wsc-v0-res.json

@@ -0,0 +1 @@
+{"results": {"wsc": {"acc": 0.5480769230769231, "acc_stderr": 0.049038186969314335}}, "versions": {"wsc": 0}}

lm-evaluation-harness/tests/testdata/wsc273-v0-loglikelihood

@@ -0,0 +1 @@
+26450d414c4581feb51a09882080e7a9b95882e7eab47b1751a4a6024b5a60ee

lm-evaluation-harness/tests/testdata/wsc273-v0-res.json

@@ -0,0 +1 @@
+{"results": {"wsc273": {"acc": 0.5164835164835165, "acc_stderr": 0.0303004740355766}}, "versions": {"wsc273": 0}}

lmms-eval/examples/models/instructblip.sh

@@ -0,0 +1,19 @@
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+python3 -m pip install transformers --upgrade;
+
+CKPT_PATH=$1
+TASK=$2
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+    --model instructblip \
+    --model_args pretrained=$CKPT_PATH \
+    --tasks $TASK \
+    --batch_size 1 \
+    --log_samples \
+    --log_samples_suffix instructblip \
+    --output_path ./logs/ 

lmms-eval/examples/models/internvl1.5.sh

@@ -0,0 +1,24 @@
+# First you need to fork [`InternVL`](https://github.com/OpenGVLab/InternVL)
+
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+cd /path/to/InternVL/internvl_chat
+python3 -m pip install -e .;
+
+python3 -m pip install flash-attn==2.3.6 --no-build-isolation;
+
+
+TASK=$1
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+    --model internvl \
+    --model_args pretrained="OpenGVLab/InternVL-Chat-V1-5"\
+    --tasks $TASK \
+    --batch_size 1 \
+    --log_samples \
+    --log_samples_suffix $TASK_SUFFIX \
+    --output_path ./logs/ 

lmms-eval/examples/models/llava_onevision.sh

@@ -0,0 +1,10 @@
+export HF_HOME="~/.cache/huggingface"
+
+# pip install git+https://github.com/LLaVA-VL/LLaVA-NeXT.git
+# pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+
+accelerate launch --num_processes=8 --main_process_port 12399 -m lmms_eval \
+    --model=llava_onevision \
+    --model_args=pretrained=lmms-lab/llava-onevision-qwen2-7b-ov,conv_template=qwen_1_5,device_map=auto,model_name=llava_qwen \
+    --tasks=ai2d,chartqa,docvqa_val,mmmu_pro \
+    --batch_size=1

lmms-eval/examples/models/llava_video.sh

@@ -0,0 +1,13 @@
+export HF_HOME="~/.cache/huggingface"
+
+# pip install git+https://github.com/LLaVA-VL/LLaVA-NeXT.git
+# pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+    --model llava_vid \
+    --model_args pretrained=lmms-lab/LLaVA-NeXT-Video-32B-Qwen,conv_template=qwen_1_5,video_decode_backend=decord,max_frames_num=32,mm_spatial_pool_mode=average,mm_newline_position=grid,mm_resampler_location=after \
+    --tasks videomme \
+    --batch_size 1 \
+    --log_samples \
+    --log_samples_suffix llava_vid_32B \
+    --output_path ./logs/

lmms-eval/examples/models/mplug_owl.sh

@@ -0,0 +1,18 @@
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+# It has to use an old transformers version to run
+python3 -m pip install av sentencepiece protobuf==3.20 transformers==4.28.1 einops;
+
+TASK=$1
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+    --model mplug_owl_video \
+    --tasks $TASK \
+    --batch_size 1 \
+    --log_samples \
+    --log_samples_suffix $TASK_SUFFIX \
+    --output_path ./logs/ 

lmms-eval/examples/models/openai_compatible.sh

@@ -0,0 +1,11 @@
+export HF_HOME="~/.cache/huggingface"
+export AZURE_OPENAI_API_KEY=""
+export AZURE_OPENAI_API_BASE=""
+export AZURE_OPENAI_API_VERSION="2023-07-01-preview"
+
+# pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+python3 -m lmms_eval \
+    --model openai_compatible \
+    --model_args model_version=gpt-4o-2024-11-20,azure_openai=True \
+    --tasks mme,mmmu_val \
+    --batch_size 1

lmms-eval/examples/models/plm.sh

@@ -0,0 +1,44 @@
+# # STEP # 1: Install lmms-eval
+# pip install lmms-eval
+
+# # STEP # 2: Install perception_models (Details at https://github.com/facebookresearch/perception_models)
+# git clone https://github.com/facebookresearch/perception_models.git
+# cd perception_models
+
+# conda create --name perception_models python=3.12
+# conda activate perception_models
+
+# # Install PyTorch
+# pip install torch==2.5.1 torchvision==0.20.1 torchaudio==2.5.1 xformers --index-url https://download.pytorch.org/whl/cu124
+
+# # We use torchcodec for decoding videos into PyTorch tensors
+# conda install ffmpeg -c conda-forge
+# pip install torchcodec==0.1 --index-url=https://download.pytorch.org/whl/cu124
+
+# pip install -e .
+
+
+# Use facebook/Perception-LM-1B for 1B parameters model and facebook/Perception-LM-8B for 8B parameters model.
+CHECKPOINTS_PATH=facebook/Perception-LM-3B
+
+# Define the tasks you want to evaluate PLM on. We support all the tasks present in lmms-eval, however have tested the following tasks with our models.
+ALL_TASKS=(
+    "docvqa" "chartqa" "textvqa" "infovqa" "ai2d_no_mask" "ok_vqa" "vizwiz_vqa" "mme"
+    "realworldqa" "pope" "mmmu" "ocrbench" "coco_karpathy_val" "nocaps" "vqav2_val"
+    "mvbench" "videomme" "vatex_test" "egoschema" "egoschema_subset" "mlvu_dev"
+    "tempcompass_multi_choice" "perceptiontest_val_mc" "perceptiontest_test_mc"
+)
+
+# We select one image and one video task as an example.
+SELECTED_TASK="textvqa,videomme"
+
+# After specifying the task/tasks to evaluate, run the following command to start the evaluation.
+accelerate launch --num_processes=8 \
+-m lmms_eval \
+--model plm \
+--model_args pretrained=$CHECKPOINTS_PATH,max_tokens=11264 \
+--tasks $SELECTED_TASK \
+--batch_size 1 \
+--log_samples \
+--log_samples_suffix plm \
+--output_path plm_reproduce

lmms-eval/examples/models/qwen25vl.sh

@@ -0,0 +1,18 @@
+# Run and exactly reproduce qwen2vl results!
+# mme as an example
+export HF_HOME="~/.cache/huggingface"
+# pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+# pip3 install qwen_vl_utils
+# use `interleave_visuals=True` to control the visual token position, currently only for mmmu_val and mmmu_pro (and potentially for other interleaved image-text tasks), please do not use it unless you are sure about the operation details.
+
+# accelerate launch --num_processes=8 --main_process_port=12346 -m lmms_eval \
+#     --model qwen2_vl \
+#     --model_args=pretrained=Qwen/Qwen2-VL-7B-Instruct,max_pixels=12845056,attn_implementation=flash_attention_2,interleave_visuals=True \
+#     --tasks mmmu_pro \
+#     --batch_size 1
+
+accelerate launch --num_processes=8 --main_process_port=12346 -m lmms_eval \
+    --model qwen2_5_vl \
+    --model_args=pretrained=Qwen/Qwen2.5-VL-7B-Instruct,max_pixels=12845056,attn_implementation=flash_attention_2,interleave_visuals=False \
+    --tasks mme \
+    --batch_size 1

lmms-eval/examples/models/qwen2vl.sh

@@ -0,0 +1,8 @@
+# Run and exactly reproduce qwen2vl results!
+# mme as an example
+pip3 install qwen_vl_utils
+accelerate launch --num_processes=8 --main_process_port=12345 -m lmms_eval \
+    --model qwen2_vl \
+    --model_args=pretrained=Qwen/Qwen2-VL-7B-Instruct,max_pixels=2359296 \
+    --tasks mme,gsm8k_cot_self_consistency,mmmu_val_reasoning,mmmu_val_pass64 \
+    --batch_size 1 --log_samples --log_samples_suffix reproduce --output_path ./logs/

lmms-eval/examples/models/sglang.sh

@@ -0,0 +1,12 @@
+export HF_HOME="~/.cache/huggingface"
+# pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+
+python3 -m lmms_eval \
+	--model=llava_sglang \
+	--model_args=pretrained=lmms-lab/llava-next-72b,tokenizer=lmms-lab/llavanext-qwen-tokenizer,conv_template=chatml-llava,tp_size=8,parallel=8 \
+	--tasks=mme \
+	--batch_size=1 \
+	--log_samples \
+	--log_samples_suffix=llava_qwen \
+	--output_path=./logs/ \
+	--verbosity=INFO

lmms-eval/examples/models/slime.sh

@@ -0,0 +1,23 @@
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+git clone https://github.com/yfzhang114/SliME.git
+
+cd SliME
+pip install --upgrade pip  # enable PEP 660 support
+pip install -e .
+cd ..
+
+TASK=$1
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+    --model slime \
+    --tasks $TASK \
+    --model_args pretrained="yifanzhang114/SliME-Llama3-8B,conv_template=llama3,model_name=slime" \
+    --batch_size 1 \
+    --log_samples \
+    --log_samples_suffix $TASK_SUFFIX \
+    --output_path ./logs/ 

lmms-eval/examples/models/tensor_parallel.sh

@@ -0,0 +1,12 @@
+export HF_HOME="~/.cache/huggingface"
+# pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+
+python3 -m lmms_eval \
+    --model=llava \
+    --model_args=pretrained=lmms-lab/llava-next-72b,conv_template=qwen_1_5,device_map=auto,model_name=llava_qwen \
+    --tasks=pope,vizwiz_vqa_val,scienceqa_img \
+    --batch_size=1 \
+    --log_samples \
+    --log_samples_suffix=llava_qwen \
+    --output_path="./logs/" \
+    --wandb_args=project=lmms-eval,job_type=eval,entity=llava-vl

lmms-eval/examples/models/tinyllava.sh

@@ -0,0 +1,25 @@
+# install lmms_eval without building dependencies
+cd lmms_eval;
+pip install --no-deps -U -e .
+
+# install TinyLLaVA without building dependencies
+cd ..
+git clone https://github.com/TinyLLaVA/TinyLLaVA_Factory
+cd TinyLLaVA_Factory
+pip install --no-deps -U -e .
+
+# install all the requirements that require for reproduce llava results
+pip install torch==2.0.1 torchvision==0.15.2 --index-url https://download.pytorch.org/whl/cu118
+pip install -r tinyllava_repr_requirements.txt
+
+# Run and reproduce tinyllava best results!
+accelerate launch \
+    --num_processes=1 \
+    -m lmms_eval \
+    --model tinyllava \
+    --model_args pretrained=tinyllava/TinyLLaVA-Phi-2-SigLIP-3.1B,conv_mode=phi \
+    --tasks vqav2,gqa,scienceqa_img,textvqa,mmvet,pope,mme,mmmu_val \
+    --batch_size 1 \
+    --log_samples \
+    --log_samples_suffix tinyllava-phi2-siglip-3.1b \
+    --output_path ./logs/

lmms-eval/examples/models/video_chatgpt.sh

@@ -0,0 +1,17 @@
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+python3 -m pip install sentencepiece av;
+
+TASK=$1
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+    --model video_chatgpt \
+    --tasks $TASK \
+    --batch_size 1 \
+    --log_samples \
+    --log_samples_suffix $TASK_SUFFIX \
+    --output_path ./logs/ 

lmms-eval/examples/models/video_llava.sh

@@ -0,0 +1,19 @@
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+python3 -m pip install transformers --upgrade;
+python3 -m pip install av sentencepiece;
+
+
+TASK=$1
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+    --model video_llava \
+    --tasks $TASK \
+    --batch_size 1 \
+    --log_samples \
+    --log_samples_suffix $TASK_SUFFIX \
+    --output_path ./logs/

lmms-eval/examples/models/vllm_qwen2vl.sh

@@ -0,0 +1,19 @@
+
+# pip3 install vllm
+# pip3 install qwen_vl_utils
+
+# cd ~/prod/lmms-eval-public
+# pip3 install -e .
+export NCCL_BLOCKING_WAIT=1
+export NCCL_TIMEOUT=18000000
+export NCCL_DEBUG=DEBUG
+
+python3 -m lmms_eval \
+    --model vllm \
+    --model_args model=Qwen/Qwen2-VL-7B-Instruct,tensor_parallel_size=4 \
+    --tasks mme,gsm8k_cot_self_consistency,mmmu_val \
+    --batch_size 64 \
+    --log_samples \
+    --log_samples_suffix vllm \
+    --output_path ./logs \
+    --limit=64

lmms-eval/examples/models/xai_grok.sh

@@ -0,0 +1,10 @@
+export HF_HOME="~/.cache/huggingface"
+export OPENAI_API_KEY="xai-xxxxxxxxxx"
+export OPENAI_API_BASE="https://api.x.ai/v1"
+
+
+python3 -m lmms_eval \
+    --model openai_compatible \
+    --model_args model_version=grok-2-vision-1212 \
+    --tasks ai2d,chartqa,docvqa_val,mathvista_testmini,mmmu_pro \
+    --batch_size 1

lmms-eval/examples/models/xcomposer.sh

@@ -0,0 +1,32 @@
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+
+python3 -m pip install flash-attn --no-build-isolation;
+python3 -m pip install torchvision einops timm sentencepiece;
+
+TASK=$1
+MODALITY=$2
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+# For Xcomposer2d5
+accelerate launch --num_processes 8 --main_process_port 10000 -m lmms_eval \
+    --model xcomposer2d5 \
+    --model_args pretrained="internlm/internlm-xcomposer2d5-7b",device="cuda",modality=$MODALITY\
+    --tasks $TASK \
+    --batch_size 1 \
+    --log_samples \
+    --log_samples_suffix $TASK_SUFFIX \
+    --output_path ./logs/ 
+
+# For Xcomposer-4kHD
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+    --model xcomposer2_4khd \
+    --model_args pretrained="internlm/internlm-xcomposer2-4khd-7b" \
+    --tasks $TASK \
+    --batch_size 1 \
+    --log_samples \
+    --log_samples_suffix $TASK_SUFFIX \
+    --output_path ./logs/

lmms-eval/lmms_eval/__main__.py

@@ -0,0 +1,539 @@
+import argparse
+import datetime
+import importlib
+import json
+import os
+import sys
+import traceback
+import warnings
+from functools import partial
+
+import numpy as np
+import torch
+import yaml
+
+warnings.simplefilter("ignore", category=DeprecationWarning)
+
+import hashlib
+from pathlib import Path
+from typing import Union
+
+from accelerate import Accelerator
+from accelerate.utils import InitProcessGroupKwargs
+from loguru import logger as eval_logger
+
+from lmms_eval import evaluator, utils
+from lmms_eval.api.registry import ALL_TASKS
+from lmms_eval.evaluator import request_caching_arg_to_dict
+from lmms_eval.loggers import EvaluationTracker, WandbLogger
+from lmms_eval.tasks import TaskManager
+from lmms_eval.utils import (
+    handle_non_serializable,
+    make_table,
+    simple_parse_args_string,
+)
+
+
+def _int_or_none_list_arg_type(min_len: int, max_len: int, defaults: str, value: str, split_char: str = ","):
+    def parse_value(item):
+        item = item.strip().lower()
+        if item == "none":
+            return None
+        try:
+            return int(item)
+        except ValueError:
+            raise argparse.ArgumentTypeError(f"{item} is not an integer or None")
+
+    items = [parse_value(v) for v in value.split(split_char)]
+    num_items = len(items)
+
+    if num_items == 1:
+        # Makes downstream handling the same for single and multiple values
+        items = items * max_len
+    elif num_items < min_len or num_items > max_len:
+        raise argparse.ArgumentTypeError(f"Argument requires {max_len} integers or None, separated by '{split_char}'")
+    elif num_items != max_len:
+        logging.warning(f"Argument requires {max_len} integers or None, separated by '{split_char}'. " "Missing values will be filled with defaults.")
+        default_items = [parse_value(v) for v in defaults.split(split_char)]
+        items.extend(default_items[num_items:])  # extend items list with missing defaults
+
+    return items
+
+
+def check_argument_types(parser: argparse.ArgumentParser):
+    """
+    Check to make sure all CLI args are typed, raises error if not
+    """
+    for action in parser._actions:
+        if action.dest != "help" and not action.const:
+            if action.type is None:
+                raise ValueError(f"Argument '{action.dest}' doesn't have a type specified.")
+            else:
+                continue
+
+
+def _handle_non_serializable(o):
+    if isinstance(o, np.int64) or isinstance(o, np.int32):
+        return int(o)
+    elif isinstance(o, set):
+        return list(o)
+    else:
+        return str(o)
+
+
+def parse_eval_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument("--config", default="", help="Path to a yaml file specifying all eval arguments, will ignore cli arguments if specified")
+    parser.add_argument("--model", default="hf", help="Name of model e.g. `hf`")
+    parser.add_argument(
+        "--tasks",
+        default=None,
+        help="To get full list of tasks, use the command lmms-eval --tasks list",
+    )
+    parser.add_argument(
+        "--model_args",
+        default="",
+        help="String arguments for model, e.g. `pretrained=EleutherAI/pythia-160m,dtype=float32`",
+    )
+    parser.add_argument(
+        "--launcher_args",
+        default=None,
+        help="String arguments for launcher for local llm as judge, e.g. `tp=8`, if None then no launcher will be used.",
+    )
+    parser.add_argument(
+        "--num_fewshot",
+        type=int,
+        default=None,
+        help="Number of examples in few-shot context",
+    )
+    parser.add_argument(
+        "--batch_size",
+        "-b",
+        type=str,
+        default=1,
+        metavar="auto|auto:N|N",
+        help="Acceptable values are 'auto', 'auto:N' or N, where N is an integer. Default 1.",
+    )
+    parser.add_argument(
+        "--max_batch_size",
+        type=int,
+        default=None,
+        metavar="N",
+        help="Maximal batch size to try with --batch_size auto.",
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default=None,
+        help="Device to use (e.g. cuda, cuda:0, cpu)",
+    )
+    parser.add_argument(
+        "--output_path",
+        default=None,
+        type=str,
+        metavar="= [dir/file.jsonl] [DIR]",
+        help="The path to the output file where the result metrics will be saved. If the path is a directory and log_samples is true, the results will be saved in the directory. Else the parent directory will be used.",
+    )
+    parser.add_argument(
+        "--limit",
+        type=float,
+        default=None,
+        help="Limit the number of examples per task. " "If <1, limit is a percentage of the total number of examples.",
+    )
+    parser.add_argument(
+        "--use_cache",
+        "-c",
+        type=str,
+        default=None,
+        metavar="DIR",
+        help="A path to a sqlite db file for caching model responses. `None` if not caching.",
+    )
+    parser.add_argument(
+        "--cache_requests",
+        type=str,
+        default=None,
+        choices=["true", "refresh", "delete"],
+        help="Speed up evaluation by caching the building of dataset requests. `None` if not caching.",
+    )
+    parser.add_argument(
+        "--check_integrity",
+        action="store_true",
+        help="Whether to run the relevant part of the test suite for the tasks",
+    )
+    parser.add_argument(
+        "--write_out",
+        "-w",
+        action="store_true",
+        default=False,
+        help="Prints the prompt for the first few documents.",
+    )
+    parser.add_argument(
+        "--log_samples",
+        action="store_true",
+        default=False,
+        help="If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis",
+    )
+    parser.add_argument(
+        "--wandb_log_samples",
+        action="store_true",
+        default=False,
+        help="If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis to Weights and Biases",
+    )
+    parser.add_argument(
+        "--log_samples_suffix",
+        type=str,
+        default="model_outputs",
+        help="Specify a suffix for the log_samples file name.",
+    )
+    parser.add_argument(
+        "--system_instruction",
+        type=str,
+        default=None,
+        help="System instruction to be used in the prompt",
+    )
+    parser.add_argument(
+        "--apply_chat_template",
+        action="store_true",
+        default=False,
+        help="If True, applies the chat template to the prompt",
+    )
+    parser.add_argument(
+        "--fewshot_as_multiturn",
+        action="store_true",
+        default=False,
+        help="If True, uses the fewshot as a multi-turn conversation",
+    )
+    parser.add_argument(
+        "--show_config",
+        action="store_true",
+        default=False,
+        help="If True, shows the the full config of all tasks at the end of the evaluation.",
+    )
+    parser.add_argument(
+        "--include_path",
+        type=str,
+        default=None,
+        help="Additional path to include if there are external tasks to include.",
+    )
+    parser.add_argument(
+        "--gen_kwargs",
+        default="",
+        help=("String arguments for model generation on greedy_until tasks," " e.g. `temperature=0,top_k=0,top_p=0`"),
+    )
+    parser.add_argument(
+        "--verbosity",
+        type=str,
+        default="INFO",
+        help="Log error when tasks are not registered.",
+    )
+    parser.add_argument(
+        "--wandb_args",
+        default="",
+        help="Comma separated string arguments passed to wandb.init, e.g. `project=lmms-eval,job_type=eval",
+    )
+    parser.add_argument(
+        "--timezone",
+        default="Asia/Singapore",
+        help="Timezone for datetime string, e.g. Asia/Singapore, America/New_York, America/Los_Angeles. You can check the full list via `import pytz; print(pytz.common_timezones)`",
+    )
+    parser.add_argument(
+        "--hf_hub_log_args",
+        type=str,
+        default="",
+        help="Comma separated string arguments passed to Hugging Face Hub's log function, e.g. `hub_results_org=EleutherAI,hub_repo_name=lm-eval-results`",
+    )
+    parser.add_argument(
+        "--predict_only",
+        "-x",
+        action="store_true",
+        default=False,
+        help="Use with --log_samples. Only model outputs will be saved and metrics will not be evaluated.",
+    )
+    default_seed_string = "0,1234,1234,1234"
+    parser.add_argument(
+        "--seed",
+        type=partial(_int_or_none_list_arg_type, 3, 4, default_seed_string),
+        default=default_seed_string,  # for backward compatibility
+        help=(
+            "Set seed for python's random, numpy, torch, and fewshot sampling.\n"
+            "Accepts a comma-separated list of 4 values for python's random, numpy, torch, and fewshot sampling seeds, "
+            "respectively, or a single integer to set the same seed for all four.\n"
+            f"The values are either an integer or 'None' to not set the seed. Default is `{default_seed_string}` "
+            "(for backward compatibility).\n"
+            "E.g. `--seed 0,None,8,52` sets `random.seed(0)`, `torch.manual_seed(8)`, and fewshot sampling seed to 52. "
+            "Here numpy's seed is not set since the second value is `None`.\n"
+            "E.g, `--seed 42` sets all four seeds to 42."
+        ),
+    )
+    parser.add_argument(
+        "--trust_remote_code",
+        action="store_true",
+        help="Sets trust_remote_code to True to execute code to create HF Datasets from the Hub",
+    )
+    parser.add_argument("--process_with_media", action="store_true", help="Whether you will process you dataset with audio, image. By default set to False" "In case some benchmarks need to be processed with media, set this flag to True.")
+    parser.add_argument("--force_simple", action="store_true", help="Force the evaluation to use the simple mode of the models")
+    args = parser.parse_args()
+    return args
+
+
+def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
+    default_args = parse_eval_args()
+
+    if args is None and len(sys.argv) == 1:
+        print("┌───────────────────────────────────────────────────────────────────────────────┐")
+        print("│ Please provide arguments to evaluate the model. e.g.                          │")
+        print("│ `lmms-eval --model llava --model_path liuhaotian/llava-v1.6-7b --tasks okvqa` │")
+        print("│ Use `lmms-eval --help` for more information.                                  │")
+        print("└───────────────────────────────────────────────────────────────────────────────┘")
+        sys.exit(1)
+
+    # If args were provided, override the defaults
+    if args:
+        for key, value in vars(args).items():
+            setattr(default_args, key, value)
+
+    args = default_args
+
+    if args.wandb_args:
+        if "name" not in args.wandb_args:
+            name = f"{args.model}_{args.model_args}_{utils.get_datetime_str(timezone=args.timezone)}"
+            name = utils.sanitize_long_string(name)
+            args.wandb_args += f",name={name}"
+        wandb_logger = WandbLogger(**simple_parse_args_string(args.wandb_args))
+
+    # reset logger
+    eval_logger.remove()
+    # Configure logger with detailed format including file path, function name, and line number
+    log_format = "<green>{time:YYYY-MM-DD HH:mm:ss}</green> | " "<level>{level: <8}</level> | " "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - " "<level>{message}</level>"
+    eval_logger.add(sys.stdout, colorize=True, level=args.verbosity, format=log_format)
+    eval_logger.info(f"Verbosity set to {args.verbosity}")
+    os.environ["VERBOSITY"] = args.verbosity
+
+    args_list = []
+    results_list = []
+    if args.config:
+        if not os.path.exists(args.config):
+            raise ValueError(f"Config file does not exist: {args.config}")
+
+        with open(args.config, "r") as file:
+            config_args = yaml.safe_load(file)
+        config_args = [config_args] if type(config_args) != list else config_args
+        # multiple configs, create args list first
+        for config in config_args:
+            args_copy = argparse.Namespace(**vars(args))
+            for key, value in config.items():
+                setattr(args_copy, key, value)
+            args_list.append(args_copy)
+    else:
+        args_list.append(args)
+
+    # initialize Accelerator only if not already in a distributed context
+    if torch.distributed.is_available() and torch.distributed.is_initialized():
+        accelerator = None
+        is_main_process = torch.distributed.get_rank() == 0
+    else:
+        kwargs_handler = InitProcessGroupKwargs(timeout=datetime.timedelta(seconds=60000))
+        accelerator = Accelerator(kwargs_handlers=[kwargs_handler])
+        if accelerator.is_main_process:
+            is_main_process = True
+        else:
+            is_main_process = False
+
+    for args in args_list:
+        try:
+            # if is_main_process and args.wandb_args:  # thoughtfully we should only init wandb once, instead of multiple ranks to avoid network traffics and unwanted behaviors.
+            #     wandb_logger = WandbLogger()
+
+            results, samples = cli_evaluate_single(args)
+            results_list.append(results)
+
+            if accelerator:
+                accelerator.wait_for_everyone()
+            elif torch.distributed.is_available() and torch.distributed.is_initialized():
+                torch.distributed.barrier()
+            if is_main_process and args.wandb_args:
+                try:
+                    wandb_logger.post_init(results)
+                    wandb_logger.log_eval_result()
+                    if args.wandb_log_samples and samples is not None:
+                        wandb_logger.log_eval_samples(samples)
+                except Exception as e:
+                    eval_logger.info(f"Logging to Weights and Biases failed due to {e}")
+                # wandb_logger.finish()
+
+        except Exception as e:
+            if args.verbosity == "DEBUG":
+                raise e
+            else:
+                traceback.print_exc()
+                eval_logger.error(f"Error during evaluation: {e}. Please set `--verbosity=DEBUG` to get more information.")
+                results_list.append(None)
+
+    for args, results in zip(args_list, results_list):
+        # cli_evaluate will return none if the process is not the main process (rank 0)
+        if results is not None:
+            print(f"{args.model} ({args.model_args}), gen_kwargs: ({args.gen_kwargs}), limit: {args.limit}, num_fewshot: {args.num_fewshot}, " f"batch_size: {args.batch_size}")
+            print(make_table(results))
+            if "groups" in results:
+                print(make_table(results, "groups"))
+
+    if args.wandb_args:
+        wandb_logger.run.finish()
+
+
+def cli_evaluate_single(args: Union[argparse.Namespace, None] = None) -> None:
+    selected_task_list = args.tasks.split(",") if args.tasks else None
+
+    if args.include_path is not None:
+        eval_logger.info(f"Including path: {args.include_path}")
+    task_manager = TaskManager(args.verbosity, include_path=args.include_path, model_name=args.model)
+
+    # update the evaluation tracker args with the output path and the HF token
+    if args.output_path:
+        args.hf_hub_log_args += f",output_path={args.output_path}"
+    if os.environ.get("HF_TOKEN", None):
+        args.hf_hub_log_args += f",token={os.environ.get('HF_TOKEN')}"
+
+    evaluation_tracker_args = simple_parse_args_string(args.hf_hub_log_args)
+    eval_logger.info(f"Evaluation tracker args: {evaluation_tracker_args}")
+
+    evaluation_tracker = EvaluationTracker(**evaluation_tracker_args)
+
+    if args.predict_only:
+        args.log_samples = True
+    if (args.log_samples or args.predict_only) and not args.output_path:
+        raise ValueError("Specify --output_path if providing --log_samples or --predict_only")
+
+    if args.fewshot_as_multiturn and args.apply_chat_template is False:
+        raise ValueError("If fewshot_as_multiturn is set, apply_chat_template must be set to True.")
+
+    if (args.num_fewshot is None or args.num_fewshot == 0) and args.fewshot_as_multiturn:
+        raise ValueError("If fewshot_as_multiturn is set, num_fewshot must be greater than 0.")
+
+    if args.include_path is not None:
+        eval_logger.info(f"Including path: {args.include_path}")
+
+    if "push_samples_to_hub" in evaluation_tracker_args and not args.log_samples:
+        eval_logger.warning("Pushing samples to the Hub requires --log_samples to be set. Samples will not be pushed to the Hub.")
+
+    if args.limit:
+        eval_logger.warning(" --limit SHOULD ONLY BE USED FOR TESTING." "REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")
+
+    if os.environ.get("LMMS_EVAL_PLUGINS", None):
+        args.include_path = [args.include_path] if args.include_path else []
+        for plugin in os.environ["LMMS_EVAL_PLUGINS"].split(","):
+            package_tasks_location = importlib.util.find_spec(f"{plugin}.tasks").submodule_search_locations[0]
+            args.include_path.append(package_tasks_location)
+
+    if args.tasks is None:
+        eval_logger.error("Need to specify task to evaluate.")
+        sys.exit()
+    elif args.tasks == "list":
+        eval_logger.info("Available Tasks:\n - {}".format(f"\n - ".join(sorted(task_manager.all_tasks))))
+        sys.exit()
+    elif args.tasks == "list_groups":
+        eval_logger.info(task_manager.list_all_tasks(list_subtasks=False, list_tags=False))
+        sys.exit()
+    elif args.tasks == "list_tags":
+        eval_logger.info(task_manager.list_all_tasks(list_groups=False, list_subtasks=False))
+        sys.exit()
+    elif args.tasks == "list_subtasks":
+        eval_logger.info(task_manager.list_all_tasks(list_groups=False, list_tags=False))
+        sys.exit()
+    else:
+        if os.path.isdir(args.tasks):
+            import glob
+
+            task_names = []
+            yaml_path = os.path.join(args.tasks, "*.yaml")
+            for yaml_file in glob.glob(yaml_path):
+                config = utils.load_yaml_config(yaml_file)
+                task_names.append(config)
+        else:
+            task_list = args.tasks.split(",")
+            task_names = task_manager.match_tasks(task_list)
+            for task in [task for task in task_list if task not in task_names]:
+                if os.path.isfile(task):
+                    config = utils.load_yaml_config(task)
+                    task_names.append(config)
+            task_missing = [task for task in task_list if task not in task_names and "*" not in task]  # we don't want errors if a wildcard ("*") task name was used
+
+            if task_missing:
+                missing = ", ".join(task_missing)
+                eval_logger.error(
+                    f"Tasks were not found: {missing}\n" f"{utils.SPACING}Try `lmms-eval --tasks list` for list of available tasks",
+                )
+                raise ValueError(
+                    f"Tasks not found: {missing}. Try `lmms-eval --tasks {{list_groups,list_subtasks,list_tags,list}}` to list out all available names for task groupings; only (sub)tasks; tags; or all of the above, or pass '--verbosity DEBUG' to troubleshoot task registration issues."
+                )
+
+    eval_logger.info(f"Selected Tasks: {task_names}")
+    request_caching_args = request_caching_arg_to_dict(cache_requests=args.cache_requests)
+    datetime_str = utils.get_datetime_str(timezone=args.timezone)
+
+    results = evaluator.simple_evaluate(
+        model=args.model,
+        model_args=args.model_args,
+        tasks=task_names,
+        num_fewshot=args.num_fewshot,
+        batch_size=args.batch_size,
+        max_batch_size=args.max_batch_size,
+        device=args.device,
+        use_cache=args.use_cache,
+        limit=args.limit,
+        check_integrity=args.check_integrity,
+        write_out=args.write_out,
+        log_samples=args.log_samples,
+        evaluation_tracker=evaluation_tracker,
+        system_instruction=args.system_instruction,
+        apply_chat_template=args.apply_chat_template,
+        fewshot_as_multiturn=args.fewshot_as_multiturn,
+        gen_kwargs=args.gen_kwargs,
+        task_manager=task_manager,
+        verbosity=args.verbosity,
+        predict_only=args.predict_only,
+        random_seed=args.seed[0],
+        numpy_random_seed=args.seed[1],
+        torch_random_seed=args.seed[2],
+        fewshot_random_seed=args.seed[3],
+        cli_args=args,
+        datetime_str=datetime_str,
+        distributed_executor_backend="torchrun" if (torch.distributed.is_available() and torch.distributed.is_initialized()) else "accelerate",
+        force_simple=args.force_simple,
+        launcher_args=args.launcher_args,
+        **request_caching_args,
+    )
+
+    if results is not None:
+        if args.log_samples:
+            samples = results.pop("samples")
+        else:
+            samples = None
+        dumped = json.dumps(results, indent=4, default=_handle_non_serializable)
+        if args.show_config:
+            print(dumped)
+
+        batch_sizes = ",".join(map(str, results["config"]["batch_sizes"]))
+
+        evaluation_tracker.save_results_aggregated(results=results, samples=samples if args.log_samples else None, datetime_str=datetime_str)
+
+        if args.log_samples:
+            for task_name, config in results["configs"].items():
+                evaluation_tracker.save_results_samples(task_name=task_name, samples=samples[task_name])
+
+        if evaluation_tracker.push_results_to_hub or evaluation_tracker.push_samples_to_hub:
+            evaluation_tracker.recreate_metadata_card()
+
+        return results, samples
+    return None, None
+
+
+def print_results(args, results):
+    print(f"{args.model} ({args.model_args}),\ngen_kwargs: ({args.gen_kwargs}),\nlimit: {args.limit},\nnum_fewshot: {args.num_fewshot},\nbatch_size: {args.batch_size}")
+    print(evaluator.make_table(results))
+    if "groups" in results:
+        print(evaluator.make_table(results, "groups"))
+
+
+if __name__ == "__main__":
+    cli_evaluate()

lmms-eval/lmms_eval/api/filter.py

@@ -0,0 +1,54 @@
+from dataclasses import dataclass
+from typing import List
+
+from datasets import Dataset
+
+from lmms_eval.api.instance import Instance
+
+
+class Filter:
+    """
+    Filter classes operate on a per-task level.
+    They take all model outputs (`instance.resps` for all `task.instances`)
+    across all instances of a task, and perform operations.
+    In a single run, one can configure any number of separate filters or lists of filters.
+
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        """
+        Can define custom behavior here, if an individual instantiation of a Filter class should have state.
+        """
+
+    def apply(self, resps, docs):
+        """
+        Defines the operation to perform on a list of the `inst.resps` properties of `Instance` objects.
+        Should return the list of (filtered) response lists *in the same order as they were input*, e.g.
+        if pass in [<inst.resps for instance 0>, <inst.resps for instance 1>] should return
+        [<filtered resps for instance 0>, <filtered resps for instance 1>]
+        """
+        return resps
+
+
+@dataclass
+class FilterEnsemble:
+    """
+    FilterEnsemble creates a pipeline applying multiple filters.
+    Its intended usage is to stack multiple post-processing steps in order.
+    `task.apply_filters` should use a list of FilterEnsemble classes that it stores, to apply each
+    pipeline separately.
+    """
+
+    name: str
+    filters: List[Filter]
+
+    def apply(self, instances: List[Instance], docs: List[Dataset]) -> None:
+        resps = [inst.resps for inst in instances]  # operate just on the model responses
+        for f in self.filters:
+            # apply filters in sequence
+            resps = f.apply(resps, docs)
+
+        # add the end results after filtering to filtered_requests of their respective source instances.
+        # has key `self.name`: each FilterEnsemble applied in a given run should use a different name.
+        for inst, resp in zip(instances, resps):
+            inst.filtered_resps[self.name] = resp

lmms-eval/lmms_eval/api/group.py

@@ -0,0 +1,104 @@
+import abc
+from dataclasses import asdict, dataclass
+from inspect import getsource
+from typing import Any, Callable, List, Optional, Union
+
+
+@dataclass
+class AggMetricConfig(dict):
+    metric: Optional[str] = None
+    aggregation: Optional[str] = "mean"
+    weight_by_size: Optional[str] = False
+    # list of filter names which should be incorporated into the aggregated metric.
+    filter_list: Optional[Union[str, list]] = "none"
+
+    def __post_init__(self):
+        if self.aggregation != "mean" and not callable(self.aggregation):
+            raise ValueError(f"Currently, 'mean' is the only pre-defined aggregation across groups' subtasks. Got '{self.aggregation}'.")
+
+        if isinstance(self.filter_list, str):
+            self.filter_list = [self.filter_list]
+
+
+@dataclass
+class GroupConfig(dict):
+    group: Optional[str] = None
+    group_alias: Optional[str] = None
+    task: Optional[Union[str, list]] = None
+    aggregate_metric_list: Optional[Union[List[AggMetricConfig], AggMetricConfig, dict]] = None
+    metadata: Optional[dict] = None  # by default, not used in the code. allows for users to pass arbitrary info to tasks
+
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def __setitem__(self, item, value):
+        return setattr(self, item, value)
+
+    def __post_init__(self):
+        if self.aggregate_metric_list is not None:
+            if isinstance(self.aggregate_metric_list, dict):
+                self.aggregate_metric_list = [self.aggregate_metric_list]
+
+            self.aggregate_metric_list = [AggMetricConfig(**item) if isinstance(item, dict) else item for item in self.aggregate_metric_list]
+
+    def to_dict(self, keep_callable: bool = False) -> dict:
+        """dumps the current config as a dictionary object, as a printable format.
+        null fields will not be printed.
+        Used for dumping results alongside full task configuration
+
+        :return: dict
+            A printable dictionary version of the TaskConfig object.
+
+        # TODO: should any default value in the TaskConfig not be printed?
+        """
+        cfg_dict = asdict(self)
+        # remove values that are `None`
+        for k, v in list(cfg_dict.items()):
+            if callable(v):
+                cfg_dict[k] = self.serialize_function(v, keep_callable=keep_callable)
+        return cfg_dict
+
+    def serialize_function(self, value: Union[Callable, str], keep_callable=False) -> Union[Callable, str]:
+        """Serializes a given function or string.
+
+        If 'keep_callable' is True, the original callable is returned.
+        Otherwise, attempts to return the source code of the callable using 'getsource'.
+        """
+        if keep_callable:
+            return value
+        else:
+            try:
+                return getsource(value)
+            except (TypeError, OSError):
+                return str(value)
+
+
+class ConfigurableGroup(abc.ABC):
+    def __init__(
+        self,
+        config: Optional[dict] = None,
+    ) -> None:
+        self._config = GroupConfig(**config)
+
+    @property
+    def group(self):
+        return self._config.group
+
+    @property
+    def group_alias(self):
+        return self._config.group_alias
+
+    @property
+    def version(self):
+        return self._config.version
+
+    @property
+    def config(self):
+        return self._config.to_dict()
+
+    @property
+    def group_name(self) -> Any:
+        return self._config.group
+
+    def __repr__(self):
+        return f"ConfigurableGroup(group={self.group}," f"group_alias={self.group_alias})"