Upload folder using huggingface_hub

20250731_120452-unified-expand/configs/20250731_120452_340278.py

@@ -0,0 +1,1976 @@
+datasets = [
+    dict(
+        abbr='openai_humaneval',
+        eval_cfg=dict(
+            evaluator=dict(
+                type='ais_bench.benchmark.datasets.HumanEvalEvaluator'),
+            k=[
+                1,
+                10,
+                100,
+            ],
+            pred_postprocessor=dict(
+                type='ais_bench.benchmark.datasets.humaneval_postprocess_v2')),
+        infer_cfg=dict(
+            inferencer=dict(
+                type='ais_bench.benchmark.openicl.icl_inferencer.GenInferencer'
+            ),
+            prompt_template=dict(
+                template=
+                'You are an intelligent programming assistant to produce Python algorithmic solutions.\nCan you complete the following Python function?\n```python\n{prompt}\n```',
+                type=
+                'ais_bench.benchmark.openicl.icl_prompt_template.PromptTemplate'
+            ),
+            retriever=dict(
+                type='ais_bench.benchmark.openicl.icl_retriever.ZeroRetriever')
+        ),
+        path='ais_bench/datasets/humaneval/human-eval-v2-20210705.jsonl',
+        reader_cfg=dict(
+            input_columns=[
+                'prompt',
+            ],
+            output_column='task_id',
+            train_split='test'),
+        type='ais_bench.benchmark.datasets.HumanevalDataset'),
+]
+eval = dict(
+    partitioner=dict(
+        out_dir='outputs/default/20250731_120452/results/',
+        type='ais_bench.benchmark.partitioners.naive.NaivePartitioner'),
+    runner=dict(
+        debug=True,
+        max_num_workers=1,
+        max_workers_per_gpu=1,
+        task=dict(
+            dump_details=True,
+            type='ais_bench.benchmark.tasks.openicl_eval.OpenICLEvalTask'),
+        type='ais_bench.benchmark.runners.local.LocalRunner'))
+infer = dict(
+    partitioner=dict(
+        out_dir='outputs/default/20250731_120452/predictions/',
+        type='ais_bench.benchmark.partitioners.naive.NaivePartitioner'),
+    runner=dict(
+        debug=True,
+        disable_cb=False,
+        max_num_workers=1,
+        num_prompts=None,
+        task=dict(
+            type='ais_bench.benchmark.tasks.openicl_infer.OpenICLInferTask'),
+        type='ais_bench.benchmark.runners.local_api.LocalAPIRunner'))
+models = [
+    dict(
+        abbr='vllm-api-general-chat',
+        attr='service',
+        batch_size=256,
+        generation_kwargs=dict(
+            repetition_penalty=1.03,
+            seed=None,
+            temperature=0.6,
+            top_k=64,
+            top_p=0.95),
+        host_ip='127.0.0.1',
+        host_port=8688,
+        max_out_len=16384,
+        model='/mnt/disk7/yiliu4/DeepSeek-R1-0528-G2-2nd/',
+        path='',
+        request_rate=0,
+        retry=2,
+        trust_remote_code=True,
+        type='ais_bench.benchmark.models.VLLMCustomAPIChat'),
+]
+summarizer = dict(summary_groups=[
+    dict(
+        name='agieval-chinese',
+        subsets=[
+            'agieval-gaokao-chinese',
+            'agieval-gaokao-english',
+            'agieval-gaokao-geography',
+            'agieval-gaokao-history',
+            'agieval-gaokao-biology',
+            'agieval-gaokao-chemistry',
+            'agieval-gaokao-physics',
+            'agieval-gaokao-mathqa',
+            'agieval-logiqa-zh',
+            'agieval-jec-qa-kd',
+            'agieval-jec-qa-ca',
+            'agieval-gaokao-mathcloze',
+        ]),
+    dict(
+        name='agieval-english',
+        subsets=[
+            'agieval-lsat-ar',
+            'agieval-lsat-lr',
+            'agieval-lsat-rc',
+            'agieval-logiqa-en',
+            'agieval-sat-math',
+            'agieval-sat-en',
+            'agieval-sat-en-without-passage',
+            'agieval-aqua-rat',
+            'agieval-math',
+        ]),
+    dict(
+        name='agieval-gaokao',
+        subsets=[
+            'agieval-gaokao-chinese',
+            'agieval-gaokao-english',
+            'agieval-gaokao-geography',
+            'agieval-gaokao-history',
+            'agieval-gaokao-biology',
+            'agieval-gaokao-chemistry',
+            'agieval-gaokao-physics',
+            'agieval-gaokao-mathqa',
+            'agieval-gaokao-mathcloze',
+        ]),
+    dict(
+        name='agieval',
+        subsets=[
+            'agieval-gaokao-chinese',
+            'agieval-gaokao-english',
+            'agieval-gaokao-geography',
+            'agieval-gaokao-history',
+            'agieval-gaokao-biology',
+            'agieval-gaokao-chemistry',
+            'agieval-gaokao-physics',
+            'agieval-gaokao-mathqa',
+            'agieval-logiqa-zh',
+            'agieval-lsat-ar',
+            'agieval-lsat-lr',
+            'agieval-lsat-rc',
+            'agieval-logiqa-en',
+            'agieval-sat-math',
+            'agieval-sat-en',
+            'agieval-sat-en-without-passage',
+            'agieval-aqua-rat',
+            'agieval-jec-qa-kd',
+            'agieval-jec-qa-ca',
+            'agieval-gaokao-mathcloze',
+            'agieval-math',
+        ]),
+    dict(
+        name='mmlu-humanities',
+        subsets=[
+            'lukaemon_mmlu_formal_logic',
+            'lukaemon_mmlu_high_school_european_history',
+            'lukaemon_mmlu_high_school_us_history',
+            'lukaemon_mmlu_high_school_world_history',
+            'lukaemon_mmlu_international_law',
+            'lukaemon_mmlu_jurisprudence',
+            'lukaemon_mmlu_logical_fallacies',
+            'lukaemon_mmlu_moral_disputes',
+            'lukaemon_mmlu_moral_scenarios',
+            'lukaemon_mmlu_philosophy',
+            'lukaemon_mmlu_prehistory',
+            'lukaemon_mmlu_professional_law',
+            'lukaemon_mmlu_world_religions',
+        ]),
+    dict(
+        name='mmlu-stem',
+        subsets=[
+            'lukaemon_mmlu_abstract_algebra',
+            'lukaemon_mmlu_anatomy',
+            'lukaemon_mmlu_astronomy',
+            'lukaemon_mmlu_college_biology',
+            'lukaemon_mmlu_college_chemistry',
+            'lukaemon_mmlu_college_computer_science',
+            'lukaemon_mmlu_college_mathematics',
+            'lukaemon_mmlu_college_physics',
+            'lukaemon_mmlu_computer_security',
+            'lukaemon_mmlu_conceptual_physics',
+            'lukaemon_mmlu_electrical_engineering',
+            'lukaemon_mmlu_elementary_mathematics',
+            'lukaemon_mmlu_high_school_biology',
+            'lukaemon_mmlu_high_school_chemistry',
+            'lukaemon_mmlu_high_school_computer_science',
+            'lukaemon_mmlu_high_school_mathematics',
+            'lukaemon_mmlu_high_school_physics',
+            'lukaemon_mmlu_high_school_statistics',
+            'lukaemon_mmlu_machine_learning',
+        ]),
+    dict(
+        name='mmlu-social-science',
+        subsets=[
+            'lukaemon_mmlu_econometrics',
+            'lukaemon_mmlu_high_school_geography',
+            'lukaemon_mmlu_high_school_government_and_politics',
+            'lukaemon_mmlu_high_school_macroeconomics',
+            'lukaemon_mmlu_high_school_microeconomics',
+            'lukaemon_mmlu_high_school_psychology',
+            'lukaemon_mmlu_human_sexuality',
+            'lukaemon_mmlu_professional_psychology',
+            'lukaemon_mmlu_public_relations',
+            'lukaemon_mmlu_security_studies',
+            'lukaemon_mmlu_sociology',
+            'lukaemon_mmlu_us_foreign_policy',
+        ]),
+    dict(
+        name='mmlu-other',
+        subsets=[
+            'lukaemon_mmlu_business_ethics',
+            'lukaemon_mmlu_clinical_knowledge',
+            'lukaemon_mmlu_college_medicine',
+            'lukaemon_mmlu_global_facts',
+            'lukaemon_mmlu_human_aging',
+            'lukaemon_mmlu_management',
+            'lukaemon_mmlu_marketing',
+            'lukaemon_mmlu_medical_genetics',
+            'lukaemon_mmlu_miscellaneous',
+            'lukaemon_mmlu_nutrition',
+            'lukaemon_mmlu_professional_accounting',
+            'lukaemon_mmlu_professional_medicine',
+            'lukaemon_mmlu_virology',
+        ]),
+    dict(
+        name='mmlu',
+        subsets=[
+            'lukaemon_mmlu_formal_logic',
+            'lukaemon_mmlu_high_school_european_history',
+            'lukaemon_mmlu_high_school_us_history',
+            'lukaemon_mmlu_high_school_world_history',
+            'lukaemon_mmlu_international_law',
+            'lukaemon_mmlu_jurisprudence',
+            'lukaemon_mmlu_logical_fallacies',
+            'lukaemon_mmlu_moral_disputes',
+            'lukaemon_mmlu_moral_scenarios',
+            'lukaemon_mmlu_philosophy',
+            'lukaemon_mmlu_prehistory',
+            'lukaemon_mmlu_professional_law',
+            'lukaemon_mmlu_world_religions',
+            'lukaemon_mmlu_abstract_algebra',
+            'lukaemon_mmlu_anatomy',
+            'lukaemon_mmlu_astronomy',
+            'lukaemon_mmlu_college_biology',
+            'lukaemon_mmlu_college_chemistry',
+            'lukaemon_mmlu_college_computer_science',
+            'lukaemon_mmlu_college_mathematics',
+            'lukaemon_mmlu_college_physics',
+            'lukaemon_mmlu_computer_security',
+            'lukaemon_mmlu_conceptual_physics',
+            'lukaemon_mmlu_electrical_engineering',
+            'lukaemon_mmlu_elementary_mathematics',
+            'lukaemon_mmlu_high_school_biology',
+            'lukaemon_mmlu_high_school_chemistry',
+            'lukaemon_mmlu_high_school_computer_science',
+            'lukaemon_mmlu_high_school_mathematics',
+            'lukaemon_mmlu_high_school_physics',
+            'lukaemon_mmlu_high_school_statistics',
+            'lukaemon_mmlu_machine_learning',
+            'lukaemon_mmlu_econometrics',
+            'lukaemon_mmlu_high_school_geography',
+            'lukaemon_mmlu_high_school_government_and_politics',
+            'lukaemon_mmlu_high_school_macroeconomics',
+            'lukaemon_mmlu_high_school_microeconomics',
+            'lukaemon_mmlu_high_school_psychology',
+            'lukaemon_mmlu_human_sexuality',
+            'lukaemon_mmlu_professional_psychology',
+            'lukaemon_mmlu_public_relations',
+            'lukaemon_mmlu_security_studies',
+            'lukaemon_mmlu_sociology',
+            'lukaemon_mmlu_us_foreign_policy',
+            'lukaemon_mmlu_business_ethics',
+            'lukaemon_mmlu_clinical_knowledge',
+            'lukaemon_mmlu_college_medicine',
+            'lukaemon_mmlu_global_facts',
+            'lukaemon_mmlu_human_aging',
+            'lukaemon_mmlu_management',
+            'lukaemon_mmlu_marketing',
+            'lukaemon_mmlu_medical_genetics',
+            'lukaemon_mmlu_miscellaneous',
+            'lukaemon_mmlu_nutrition',
+            'lukaemon_mmlu_professional_accounting',
+            'lukaemon_mmlu_professional_medicine',
+            'lukaemon_mmlu_virology',
+        ]),
+    dict(
+        name='mmlu-weighted',
+        subsets=[
+            'lukaemon_mmlu_formal_logic',
+            'lukaemon_mmlu_high_school_european_history',
+            'lukaemon_mmlu_high_school_us_history',
+            'lukaemon_mmlu_high_school_world_history',
+            'lukaemon_mmlu_international_law',
+            'lukaemon_mmlu_jurisprudence',
+            'lukaemon_mmlu_logical_fallacies',
+            'lukaemon_mmlu_moral_disputes',
+            'lukaemon_mmlu_moral_scenarios',
+            'lukaemon_mmlu_philosophy',
+            'lukaemon_mmlu_prehistory',
+            'lukaemon_mmlu_professional_law',
+            'lukaemon_mmlu_world_religions',
+            'lukaemon_mmlu_abstract_algebra',
+            'lukaemon_mmlu_anatomy',
+            'lukaemon_mmlu_astronomy',
+            'lukaemon_mmlu_college_biology',
+            'lukaemon_mmlu_college_chemistry',
+            'lukaemon_mmlu_college_computer_science',
+            'lukaemon_mmlu_college_mathematics',
+            'lukaemon_mmlu_college_physics',
+            'lukaemon_mmlu_computer_security',
+            'lukaemon_mmlu_conceptual_physics',
+            'lukaemon_mmlu_electrical_engineering',
+            'lukaemon_mmlu_elementary_mathematics',
+            'lukaemon_mmlu_high_school_biology',
+            'lukaemon_mmlu_high_school_chemistry',
+            'lukaemon_mmlu_high_school_computer_science',
+            'lukaemon_mmlu_high_school_mathematics',
+            'lukaemon_mmlu_high_school_physics',
+            'lukaemon_mmlu_high_school_statistics',
+            'lukaemon_mmlu_machine_learning',
+            'lukaemon_mmlu_econometrics',
+            'lukaemon_mmlu_high_school_geography',
+            'lukaemon_mmlu_high_school_government_and_politics',
+            'lukaemon_mmlu_high_school_macroeconomics',
+            'lukaemon_mmlu_high_school_microeconomics',
+            'lukaemon_mmlu_high_school_psychology',
+            'lukaemon_mmlu_human_sexuality',
+            'lukaemon_mmlu_professional_psychology',
+            'lukaemon_mmlu_public_relations',
+            'lukaemon_mmlu_security_studies',
+            'lukaemon_mmlu_sociology',
+            'lukaemon_mmlu_us_foreign_policy',
+            'lukaemon_mmlu_business_ethics',
+            'lukaemon_mmlu_clinical_knowledge',
+            'lukaemon_mmlu_college_medicine',
+            'lukaemon_mmlu_global_facts',
+            'lukaemon_mmlu_human_aging',
+            'lukaemon_mmlu_management',
+            'lukaemon_mmlu_marketing',
+            'lukaemon_mmlu_medical_genetics',
+            'lukaemon_mmlu_miscellaneous',
+            'lukaemon_mmlu_nutrition',
+            'lukaemon_mmlu_professional_accounting',
+            'lukaemon_mmlu_professional_medicine',
+            'lukaemon_mmlu_virology',
+        ],
+        weights=dict(
+            lukaemon_mmlu_abstract_algebra=100,
+            lukaemon_mmlu_anatomy=135,
+            lukaemon_mmlu_astronomy=152,
+            lukaemon_mmlu_business_ethics=100,
+            lukaemon_mmlu_clinical_knowledge=265,
+            lukaemon_mmlu_college_biology=144,
+            lukaemon_mmlu_college_chemistry=100,
+            lukaemon_mmlu_college_computer_science=100,
+            lukaemon_mmlu_college_mathematics=100,
+            lukaemon_mmlu_college_medicine=173,
+            lukaemon_mmlu_college_physics=102,
+            lukaemon_mmlu_computer_security=100,
+            lukaemon_mmlu_conceptual_physics=235,
+            lukaemon_mmlu_econometrics=114,
+            lukaemon_mmlu_electrical_engineering=145,
+            lukaemon_mmlu_elementary_mathematics=378,
+            lukaemon_mmlu_formal_logic=126,
+            lukaemon_mmlu_global_facts=100,
+            lukaemon_mmlu_high_school_biology=310,
+            lukaemon_mmlu_high_school_chemistry=203,
+            lukaemon_mmlu_high_school_computer_science=100,
+            lukaemon_mmlu_high_school_european_history=165,
+            lukaemon_mmlu_high_school_geography=198,
+            lukaemon_mmlu_high_school_government_and_politics=193,
+            lukaemon_mmlu_high_school_macroeconomics=390,
+            lukaemon_mmlu_high_school_mathematics=270,
+            lukaemon_mmlu_high_school_microeconomics=238,
+            lukaemon_mmlu_high_school_physics=151,
+            lukaemon_mmlu_high_school_psychology=545,
+            lukaemon_mmlu_high_school_statistics=216,
+            lukaemon_mmlu_high_school_us_history=204,
+            lukaemon_mmlu_high_school_world_history=237,
+            lukaemon_mmlu_human_aging=223,
+            lukaemon_mmlu_human_sexuality=131,
+            lukaemon_mmlu_international_law=121,
+            lukaemon_mmlu_jurisprudence=108,
+            lukaemon_mmlu_logical_fallacies=163,
+            lukaemon_mmlu_machine_learning=112,
+            lukaemon_mmlu_management=103,
+            lukaemon_mmlu_marketing=234,
+            lukaemon_mmlu_medical_genetics=100,
+            lukaemon_mmlu_miscellaneous=783,
+            lukaemon_mmlu_moral_disputes=346,
+            lukaemon_mmlu_moral_scenarios=895,
+            lukaemon_mmlu_nutrition=306,
+            lukaemon_mmlu_philosophy=311,
+            lukaemon_mmlu_prehistory=324,
+            lukaemon_mmlu_professional_accounting=282,
+            lukaemon_mmlu_professional_law=1534,
+            lukaemon_mmlu_professional_medicine=272,
+            lukaemon_mmlu_professional_psychology=612,
+            lukaemon_mmlu_public_relations=110,
+            lukaemon_mmlu_security_studies=245,
+            lukaemon_mmlu_sociology=201,
+            lukaemon_mmlu_us_foreign_policy=100,
+            lukaemon_mmlu_virology=166,
+            lukaemon_mmlu_world_religions=171)),
+    dict(
+        name='cmmlu-humanities',
+        subsets=[
+            'cmmlu-arts',
+            'cmmlu-chinese_history',
+            'cmmlu-chinese_literature',
+            'cmmlu-college_law',
+            'cmmlu-global_facts',
+            'cmmlu-international_law',
+            'cmmlu-jurisprudence',
+            'cmmlu-logical',
+            'cmmlu-marxist_theory',
+            'cmmlu-philosophy',
+            'cmmlu-professional_law',
+            'cmmlu-world_history',
+            'cmmlu-world_religions',
+        ]),
+    dict(
+        name='cmmlu-stem',
+        subsets=[
+            'cmmlu-anatomy',
+            'cmmlu-astronomy',
+            'cmmlu-college_actuarial_science',
+            'cmmlu-college_engineering_hydrology',
+            'cmmlu-college_mathematics',
+            'cmmlu-college_medical_statistics',
+            'cmmlu-computer_science',
+            'cmmlu-conceptual_physics',
+            'cmmlu-electrical_engineering',
+            'cmmlu-elementary_mathematics',
+            'cmmlu-genetics',
+            'cmmlu-high_school_biology',
+            'cmmlu-high_school_chemistry',
+            'cmmlu-high_school_mathematics',
+            'cmmlu-high_school_physics',
+            'cmmlu-machine_learning',
+            'cmmlu-virology',
+        ]),
+    dict(
+        name='cmmlu-social-science',
+        subsets=[
+            'cmmlu-ancient_chinese',
+            'cmmlu-business_ethics',
+            'cmmlu-chinese_civil_service_exam',
+            'cmmlu-chinese_food_culture',
+            'cmmlu-chinese_foreign_policy',
+            'cmmlu-chinese_teacher_qualification',
+            'cmmlu-college_education',
+            'cmmlu-economics',
+            'cmmlu-education',
+            'cmmlu-elementary_chinese',
+            'cmmlu-ethnology',
+            'cmmlu-high_school_geography',
+            'cmmlu-high_school_politics',
+            'cmmlu-journalism',
+            'cmmlu-management',
+            'cmmlu-marketing',
+            'cmmlu-modern_chinese',
+            'cmmlu-professional_accounting',
+            'cmmlu-professional_psychology',
+            'cmmlu-public_relations',
+            'cmmlu-security_study',
+            'cmmlu-sociology',
+        ]),
+    dict(
+        name='cmmlu-other',
+        subsets=[
+            'cmmlu-agronomy',
+            'cmmlu-chinese_driving_rule',
+            'cmmlu-clinical_knowledge',
+            'cmmlu-college_medicine',
+            'cmmlu-computer_security',
+            'cmmlu-construction_project_management',
+            'cmmlu-elementary_commonsense',
+            'cmmlu-elementary_information_and_technology',
+            'cmmlu-food_science',
+            'cmmlu-human_sexuality',
+            'cmmlu-legal_and_moral_basis',
+            'cmmlu-nutrition',
+            'cmmlu-professional_medicine',
+            'cmmlu-sports_science',
+            'cmmlu-traditional_chinese_medicine',
+        ]),
+    dict(
+        name='cmmlu-china-specific',
+        subsets=[
+            'cmmlu-ancient_chinese',
+            'cmmlu-chinese_civil_service_exam',
+            'cmmlu-chinese_driving_rule',
+            'cmmlu-chinese_food_culture',
+            'cmmlu-chinese_foreign_policy',
+            'cmmlu-chinese_history',
+            'cmmlu-chinese_literature',
+            'cmmlu-chinese_teacher_qualification',
+            'cmmlu-construction_project_management',
+            'cmmlu-elementary_chinese',
+            'cmmlu-elementary_commonsense',
+            'cmmlu-ethnology',
+            'cmmlu-high_school_politics',
+            'cmmlu-modern_chinese',
+            'cmmlu-traditional_chinese_medicine',
+        ]),
+    dict(
+        name='cmmlu',
+        subsets=[
+            'cmmlu-agronomy',
+            'cmmlu-anatomy',
+            'cmmlu-ancient_chinese',
+            'cmmlu-arts',
+            'cmmlu-astronomy',
+            'cmmlu-business_ethics',
+            'cmmlu-chinese_civil_service_exam',
+            'cmmlu-chinese_driving_rule',
+            'cmmlu-chinese_food_culture',
+            'cmmlu-chinese_foreign_policy',
+            'cmmlu-chinese_history',
+            'cmmlu-chinese_literature',
+            'cmmlu-chinese_teacher_qualification',
+            'cmmlu-college_actuarial_science',
+            'cmmlu-college_education',
+            'cmmlu-college_engineering_hydrology',
+            'cmmlu-college_law',
+            'cmmlu-college_mathematics',
+            'cmmlu-college_medical_statistics',
+            'cmmlu-clinical_knowledge',
+            'cmmlu-college_medicine',
+            'cmmlu-computer_science',
+            'cmmlu-computer_security',
+            'cmmlu-conceptual_physics',
+            'cmmlu-construction_project_management',
+            'cmmlu-economics',
+            'cmmlu-education',
+            'cmmlu-elementary_chinese',
+            'cmmlu-elementary_commonsense',
+            'cmmlu-elementary_information_and_technology',
+            'cmmlu-electrical_engineering',
+            'cmmlu-elementary_mathematics',
+            'cmmlu-ethnology',
+            'cmmlu-food_science',
+            'cmmlu-genetics',
+            'cmmlu-global_facts',
+            'cmmlu-high_school_biology',
+            'cmmlu-high_school_chemistry',
+            'cmmlu-high_school_geography',
+            'cmmlu-high_school_mathematics',
+            'cmmlu-high_school_physics',
+            'cmmlu-high_school_politics',
+            'cmmlu-human_sexuality',
+            'cmmlu-international_law',
+            'cmmlu-journalism',
+            'cmmlu-jurisprudence',
+            'cmmlu-legal_and_moral_basis',
+            'cmmlu-logical',
+            'cmmlu-machine_learning',
+            'cmmlu-management',
+            'cmmlu-marketing',
+            'cmmlu-marxist_theory',
+            'cmmlu-modern_chinese',
+            'cmmlu-nutrition',
+            'cmmlu-philosophy',
+            'cmmlu-professional_accounting',
+            'cmmlu-professional_law',
+            'cmmlu-professional_medicine',
+            'cmmlu-professional_psychology',
+            'cmmlu-public_relations',
+            'cmmlu-security_study',
+            'cmmlu-sociology',
+            'cmmlu-sports_science',
+            'cmmlu-traditional_chinese_medicine',
+            'cmmlu-virology',
+            'cmmlu-world_history',
+            'cmmlu-world_religions',
+        ]),
+    dict(
+        name='cmmlu-weighted',
+        subsets=[
+            'cmmlu-agronomy',
+            'cmmlu-anatomy',
+            'cmmlu-ancient_chinese',
+            'cmmlu-arts',
+            'cmmlu-astronomy',
+            'cmmlu-business_ethics',
+            'cmmlu-chinese_civil_service_exam',
+            'cmmlu-chinese_driving_rule',
+            'cmmlu-chinese_food_culture',
+            'cmmlu-chinese_foreign_policy',
+            'cmmlu-chinese_history',
+            'cmmlu-chinese_literature',
+            'cmmlu-chinese_teacher_qualification',
+            'cmmlu-college_actuarial_science',
+            'cmmlu-college_education',
+            'cmmlu-college_engineering_hydrology',
+            'cmmlu-college_law',
+            'cmmlu-college_mathematics',
+            'cmmlu-college_medical_statistics',
+            'cmmlu-clinical_knowledge',
+            'cmmlu-college_medicine',
+            'cmmlu-computer_science',
+            'cmmlu-computer_security',
+            'cmmlu-conceptual_physics',
+            'cmmlu-construction_project_management',
+            'cmmlu-economics',
+            'cmmlu-education',
+            'cmmlu-elementary_chinese',
+            'cmmlu-elementary_commonsense',
+            'cmmlu-elementary_information_and_technology',
+            'cmmlu-electrical_engineering',
+            'cmmlu-elementary_mathematics',
+            'cmmlu-ethnology',
+            'cmmlu-food_science',
+            'cmmlu-genetics',
+            'cmmlu-global_facts',
+            'cmmlu-high_school_biology',
+            'cmmlu-high_school_chemistry',
+            'cmmlu-high_school_geography',
+            'cmmlu-high_school_mathematics',
+            'cmmlu-high_school_physics',
+            'cmmlu-high_school_politics',
+            'cmmlu-human_sexuality',
+            'cmmlu-international_law',
+            'cmmlu-journalism',
+            'cmmlu-jurisprudence',
+            'cmmlu-legal_and_moral_basis',
+            'cmmlu-logical',
+            'cmmlu-machine_learning',
+            'cmmlu-management',
+            'cmmlu-marketing',
+            'cmmlu-marxist_theory',
+            'cmmlu-modern_chinese',
+            'cmmlu-nutrition',
+            'cmmlu-philosophy',
+            'cmmlu-professional_accounting',
+            'cmmlu-professional_law',
+            'cmmlu-professional_medicine',
+            'cmmlu-professional_psychology',
+            'cmmlu-public_relations',
+            'cmmlu-security_study',
+            'cmmlu-sociology',
+            'cmmlu-sports_science',
+            'cmmlu-traditional_chinese_medicine',
+            'cmmlu-virology',
+            'cmmlu-world_history',
+            'cmmlu-world_religions',
+        ],
+        weights=dict({
+            'cmmlu-agronomy': 169,
+            'cmmlu-anatomy': 148,
+            'cmmlu-ancient_chinese': 164,
+            'cmmlu-arts': 160,
+            'cmmlu-astronomy': 165,
+            'cmmlu-business_ethics': 209,
+            'cmmlu-chinese_civil_service_exam': 160,
+            'cmmlu-chinese_driving_rule': 131,
+            'cmmlu-chinese_food_culture': 136,
+            'cmmlu-chinese_foreign_policy': 107,
+            'cmmlu-chinese_history': 323,
+            'cmmlu-chinese_literature': 204,
+            'cmmlu-chinese_teacher_qualification': 179,
+            'cmmlu-clinical_knowledge': 237,
+            'cmmlu-college_actuarial_science': 106,
+            'cmmlu-college_education': 107,
+            'cmmlu-college_engineering_hydrology': 106,
+            'cmmlu-college_law': 108,
+            'cmmlu-college_mathematics': 105,
+            'cmmlu-college_medical_statistics': 106,
+            'cmmlu-college_medicine': 273,
+            'cmmlu-computer_science': 204,
+            'cmmlu-computer_security': 171,
+            'cmmlu-conceptual_physics': 147,
+            'cmmlu-construction_project_management': 139,
+            'cmmlu-economics': 159,
+            'cmmlu-education': 163,
+            'cmmlu-electrical_engineering': 172,
+            'cmmlu-elementary_chinese': 252,
+            'cmmlu-elementary_commonsense': 198,
+            'cmmlu-elementary_information_and_technology': 238,
+            'cmmlu-elementary_mathematics': 230,
+            'cmmlu-ethnology': 135,
+            'cmmlu-food_science': 143,
+            'cmmlu-genetics': 176,
+            'cmmlu-global_facts': 149,
+            'cmmlu-high_school_biology': 169,
+            'cmmlu-high_school_chemistry': 132,
+            'cmmlu-high_school_geography': 118,
+            'cmmlu-high_school_mathematics': 164,
+            'cmmlu-high_school_physics': 110,
+            'cmmlu-high_school_politics': 143,
+            'cmmlu-human_sexuality': 126,
+            'cmmlu-international_law': 185,
+            'cmmlu-journalism': 172,
+            'cmmlu-jurisprudence': 411,
+            'cmmlu-legal_and_moral_basis': 214,
+            'cmmlu-logical': 123,
+            'cmmlu-machine_learning': 122,
+            'cmmlu-management': 210,
+            'cmmlu-marketing': 180,
+            'cmmlu-marxist_theory': 189,
+            'cmmlu-modern_chinese': 116,
+            'cmmlu-nutrition': 145,
+            'cmmlu-philosophy': 105,
+            'cmmlu-professional_accounting': 175,
+            'cmmlu-professional_law': 211,
+            'cmmlu-professional_medicine': 376,
+            'cmmlu-professional_psychology': 232,
+            'cmmlu-public_relations': 174,
+            'cmmlu-security_study': 135,
+            'cmmlu-sociology': 226,
+            'cmmlu-sports_science': 165,
+            'cmmlu-traditional_chinese_medicine': 185,
+            'cmmlu-virology': 169,
+            'cmmlu-world_history': 161,
+            'cmmlu-world_religions': 160
+        })),
+    dict(
+        name='ceval-stem',
+        subsets=[
+            'ceval-computer_network',
+            'ceval-operating_system',
+            'ceval-computer_architecture',
+            'ceval-college_programming',
+            'ceval-college_physics',
+            'ceval-college_chemistry',
+            'ceval-advanced_mathematics',
+            'ceval-probability_and_statistics',
+            'ceval-discrete_mathematics',
+            'ceval-electrical_engineer',
+            'ceval-metrology_engineer',
+            'ceval-high_school_mathematics',
+            'ceval-high_school_physics',
+            'ceval-high_school_chemistry',
+            'ceval-high_school_biology',
+            'ceval-middle_school_mathematics',
+            'ceval-middle_school_biology',
+            'ceval-middle_school_physics',
+            'ceval-middle_school_chemistry',
+            'ceval-veterinary_medicine',
+        ]),
+    dict(
+        name='ceval-social-science',
+        subsets=[
+            'ceval-college_economics',
+            'ceval-business_administration',
+            'ceval-marxism',
+            'ceval-mao_zedong_thought',
+            'ceval-education_science',
+            'ceval-teacher_qualification',
+            'ceval-high_school_politics',
+            'ceval-high_school_geography',
+            'ceval-middle_school_politics',
+            'ceval-middle_school_geography',
+        ]),
+    dict(
+        name='ceval-humanities',
+        subsets=[
+            'ceval-modern_chinese_history',
+            'ceval-ideological_and_moral_cultivation',
+            'ceval-logic',
+            'ceval-law',
+            'ceval-chinese_language_and_literature',
+            'ceval-art_studies',
+            'ceval-professional_tour_guide',
+            'ceval-legal_professional',
+            'ceval-high_school_chinese',
+            'ceval-high_school_history',
+            'ceval-middle_school_history',
+        ]),
+    dict(
+        name='ceval-other',
+        subsets=[
+            'ceval-civil_servant',
+            'ceval-sports_science',
+            'ceval-plant_protection',
+            'ceval-basic_medicine',
+            'ceval-clinical_medicine',
+            'ceval-urban_and_rural_planner',
+            'ceval-accountant',
+            'ceval-fire_engineer',
+            'ceval-environmental_impact_assessment_engineer',
+            'ceval-tax_accountant',
+            'ceval-physician',
+        ]),
+    dict(
+        name='ceval-hard',
+        subsets=[
+            'ceval-advanced_mathematics',
+            'ceval-discrete_mathematics',
+            'ceval-probability_and_statistics',
+            'ceval-college_chemistry',
+            'ceval-college_physics',
+            'ceval-high_school_mathematics',
+            'ceval-high_school_chemistry',
+            'ceval-high_school_physics',
+        ]),
+    dict(
+        name='ceval',
+        subsets=[
+            'ceval-computer_network',
+            'ceval-operating_system',
+            'ceval-computer_architecture',
+            'ceval-college_programming',
+            'ceval-college_physics',
+            'ceval-college_chemistry',
+            'ceval-advanced_mathematics',
+            'ceval-probability_and_statistics',
+            'ceval-discrete_mathematics',
+            'ceval-electrical_engineer',
+            'ceval-metrology_engineer',
+            'ceval-high_school_mathematics',
+            'ceval-high_school_physics',
+            'ceval-high_school_chemistry',
+            'ceval-high_school_biology',
+            'ceval-middle_school_mathematics',
+            'ceval-middle_school_biology',
+            'ceval-middle_school_physics',
+            'ceval-middle_school_chemistry',
+            'ceval-veterinary_medicine',
+            'ceval-college_economics',
+            'ceval-business_administration',
+            'ceval-marxism',
+            'ceval-mao_zedong_thought',
+            'ceval-education_science',
+            'ceval-teacher_qualification',
+            'ceval-high_school_politics',
+            'ceval-high_school_geography',
+            'ceval-middle_school_politics',
+            'ceval-middle_school_geography',
+            'ceval-modern_chinese_history',
+            'ceval-ideological_and_moral_cultivation',
+            'ceval-logic',
+            'ceval-law',
+            'ceval-chinese_language_and_literature',
+            'ceval-art_studies',
+            'ceval-professional_tour_guide',
+            'ceval-legal_professional',
+            'ceval-high_school_chinese',
+            'ceval-high_school_history',
+            'ceval-middle_school_history',
+            'ceval-civil_servant',
+            'ceval-sports_science',
+            'ceval-plant_protection',
+            'ceval-basic_medicine',
+            'ceval-clinical_medicine',
+            'ceval-urban_and_rural_planner',
+            'ceval-accountant',
+            'ceval-fire_engineer',
+            'ceval-environmental_impact_assessment_engineer',
+            'ceval-tax_accountant',
+            'ceval-physician',
+        ]),
+    dict(
+        name='ceval-weighted',
+        subsets=[
+            'ceval-computer_network',
+            'ceval-operating_system',
+            'ceval-computer_architecture',
+            'ceval-college_programming',
+            'ceval-college_physics',
+            'ceval-college_chemistry',
+            'ceval-advanced_mathematics',
+            'ceval-probability_and_statistics',
+            'ceval-discrete_mathematics',
+            'ceval-electrical_engineer',
+            'ceval-metrology_engineer',
+            'ceval-high_school_mathematics',
+            'ceval-high_school_physics',
+            'ceval-high_school_chemistry',
+            'ceval-high_school_biology',
+            'ceval-middle_school_mathematics',
+            'ceval-middle_school_biology',
+            'ceval-middle_school_physics',
+            'ceval-middle_school_chemistry',
+            'ceval-veterinary_medicine',
+            'ceval-college_economics',
+            'ceval-business_administration',
+            'ceval-marxism',
+            'ceval-mao_zedong_thought',
+            'ceval-education_science',
+            'ceval-teacher_qualification',
+            'ceval-high_school_politics',
+            'ceval-high_school_geography',
+            'ceval-middle_school_politics',
+            'ceval-middle_school_geography',
+            'ceval-modern_chinese_history',
+            'ceval-ideological_and_moral_cultivation',
+            'ceval-logic',
+            'ceval-law',
+            'ceval-chinese_language_and_literature',
+            'ceval-art_studies',
+            'ceval-professional_tour_guide',
+            'ceval-legal_professional',
+            'ceval-high_school_chinese',
+            'ceval-high_school_history',
+            'ceval-middle_school_history',
+            'ceval-civil_servant',
+            'ceval-sports_science',
+            'ceval-plant_protection',
+            'ceval-basic_medicine',
+            'ceval-clinical_medicine',
+            'ceval-urban_and_rural_planner',
+            'ceval-accountant',
+            'ceval-fire_engineer',
+            'ceval-environmental_impact_assessment_engineer',
+            'ceval-tax_accountant',
+            'ceval-physician',
+        ],
+        weights=dict({
+            'ceval-accountant': 49,
+            'ceval-advanced_mathematics': 19,
+            'ceval-art_studies': 33,
+            'ceval-basic_medicine': 19,
+            'ceval-business_administration': 33,
+            'ceval-chinese_language_and_literature': 23,
+            'ceval-civil_servant': 47,
+            'ceval-clinical_medicine': 22,
+            'ceval-college_chemistry': 24,
+            'ceval-college_economics': 55,
+            'ceval-college_physics': 19,
+            'ceval-college_programming': 37,
+            'ceval-computer_architecture': 21,
+            'ceval-computer_network': 19,
+            'ceval-discrete_mathematics': 16,
+            'ceval-education_science': 29,
+            'ceval-electrical_engineer': 37,
+            'ceval-environmental_impact_assessment_engineer': 31,
+            'ceval-fire_engineer': 31,
+            'ceval-high_school_biology': 19,
+            'ceval-high_school_chemistry': 19,
+            'ceval-high_school_chinese': 19,
+            'ceval-high_school_geography': 19,
+            'ceval-high_school_history': 20,
+            'ceval-high_school_mathematics': 18,
+            'ceval-high_school_physics': 19,
+            'ceval-high_school_politics': 19,
+            'ceval-ideological_and_moral_cultivation': 19,
+            'ceval-law': 24,
+            'ceval-legal_professional': 23,
+            'ceval-logic': 22,
+            'ceval-mao_zedong_thought': 24,
+            'ceval-marxism': 19,
+            'ceval-metrology_engineer': 24,
+            'ceval-middle_school_biology': 21,
+            'ceval-middle_school_chemistry': 20,
+            'ceval-middle_school_geography': 12,
+            'ceval-middle_school_history': 22,
+            'ceval-middle_school_mathematics': 19,
+            'ceval-middle_school_physics': 19,
+            'ceval-middle_school_politics': 21,
+            'ceval-modern_chinese_history': 23,
+            'ceval-operating_system': 19,
+            'ceval-physician': 49,
+            'ceval-plant_protection': 22,
+            'ceval-probability_and_statistics': 18,
+            'ceval-professional_tour_guide': 29,
+            'ceval-sports_science': 19,
+            'ceval-tax_accountant': 49,
+            'ceval-teacher_qualification': 44,
+            'ceval-urban_and_rural_planner': 46,
+            'ceval-veterinary_medicine': 23
+        })),
+    dict(
+        name='ceval-test-stem',
+        subsets=[
+            'ceval-test-computer_network',
+            'ceval-test-operating_system',
+            'ceval-test-computer_architecture',
+            'ceval-test-college_programming',
+            'ceval-test-college_physics',
+            'ceval-test-college_chemistry',
+            'ceval-test-advanced_mathematics',
+            'ceval-test-probability_and_statistics',
+            'ceval-test-discrete_mathematics',
+            'ceval-test-electrical_engineer',
+            'ceval-test-metrology_engineer',
+            'ceval-test-high_school_mathematics',
+            'ceval-test-high_school_physics',
+            'ceval-test-high_school_chemistry',
+            'ceval-test-high_school_biology',
+            'ceval-test-middle_school_mathematics',
+            'ceval-test-middle_school_biology',
+            'ceval-test-middle_school_physics',
+            'ceval-test-middle_school_chemistry',
+            'ceval-test-veterinary_medicine',
+        ]),
+    dict(
+        name='ceval-test-social-science',
+        subsets=[
+            'ceval-test-college_economics',
+            'ceval-test-business_administration',
+            'ceval-test-marxism',
+            'ceval-test-mao_zedong_thought',
+            'ceval-test-education_science',
+            'ceval-test-teacher_qualification',
+            'ceval-test-high_school_politics',
+            'ceval-test-high_school_geography',
+            'ceval-test-middle_school_politics',
+            'ceval-test-middle_school_geography',
+        ]),
+    dict(
+        name='ceval-test-humanities',
+        subsets=[
+            'ceval-test-modern_chinese_history',
+            'ceval-test-ideological_and_moral_cultivation',
+            'ceval-test-logic',
+            'ceval-test-law',
+            'ceval-test-chinese_language_and_literature',
+            'ceval-test-art_studies',
+            'ceval-test-professional_tour_guide',
+            'ceval-test-legal_professional',
+            'ceval-test-high_school_chinese',
+            'ceval-test-high_school_history',
+            'ceval-test-middle_school_history',
+        ]),
+    dict(
+        name='ceval-test-other',
+        subsets=[
+            'ceval-test-civil_servant',
+            'ceval-test-sports_science',
+            'ceval-test-plant_protection',
+            'ceval-test-basic_medicine',
+            'ceval-test-clinical_medicine',
+            'ceval-test-urban_and_rural_planner',
+            'ceval-test-accountant',
+            'ceval-test-fire_engineer',
+            'ceval-test-environmental_impact_assessment_engineer',
+            'ceval-test-tax_accountant',
+            'ceval-test-physician',
+        ]),
+    dict(
+        name='ceval-test-hard',
+        subsets=[
+            'ceval-test-advanced_mathematics',
+            'ceval-test-discrete_mathematics',
+            'ceval-test-probability_and_statistics',
+            'ceval-test-college_chemistry',
+            'ceval-test-college_physics',
+            'ceval-test-high_school_mathematics',
+            'ceval-test-high_school_chemistry',
+            'ceval-test-high_school_physics',
+        ]),
+    dict(
+        name='ceval-test',
+        subsets=[
+            'ceval-test-computer_network',
+            'ceval-test-operating_system',
+            'ceval-test-computer_architecture',
+            'ceval-test-college_programming',
+            'ceval-test-college_physics',
+            'ceval-test-college_chemistry',
+            'ceval-test-advanced_mathematics',
+            'ceval-test-probability_and_statistics',
+            'ceval-test-discrete_mathematics',
+            'ceval-test-electrical_engineer',
+            'ceval-test-metrology_engineer',
+            'ceval-test-high_school_mathematics',
+            'ceval-test-high_school_physics',
+            'ceval-test-high_school_chemistry',
+            'ceval-test-high_school_biology',
+            'ceval-test-middle_school_mathematics',
+            'ceval-test-middle_school_biology',
+            'ceval-test-middle_school_physics',
+            'ceval-test-middle_school_chemistry',
+            'ceval-test-veterinary_medicine',
+            'ceval-test-college_economics',
+            'ceval-test-business_administration',
+            'ceval-test-marxism',
+            'ceval-test-mao_zedong_thought',
+            'ceval-test-education_science',
+            'ceval-test-teacher_qualification',
+            'ceval-test-high_school_politics',
+            'ceval-test-high_school_geography',
+            'ceval-test-middle_school_politics',
+            'ceval-test-middle_school_geography',
+            'ceval-test-modern_chinese_history',
+            'ceval-test-ideological_and_moral_cultivation',
+            'ceval-test-logic',
+            'ceval-test-law',
+            'ceval-test-chinese_language_and_literature',
+            'ceval-test-art_studies',
+            'ceval-test-professional_tour_guide',
+            'ceval-test-legal_professional',
+            'ceval-test-high_school_chinese',
+            'ceval-test-high_school_history',
+            'ceval-test-middle_school_history',
+            'ceval-test-civil_servant',
+            'ceval-test-sports_science',
+            'ceval-test-plant_protection',
+            'ceval-test-basic_medicine',
+            'ceval-test-clinical_medicine',
+            'ceval-test-urban_and_rural_planner',
+            'ceval-test-accountant',
+            'ceval-test-fire_engineer',
+            'ceval-test-environmental_impact_assessment_engineer',
+            'ceval-test-tax_accountant',
+            'ceval-test-physician',
+        ]),
+    dict(
+        name='ceval-test-weighted',
+        subsets=[
+            'ceval-test-computer_network',
+            'ceval-test-operating_system',
+            'ceval-test-computer_architecture',
+            'ceval-test-college_programming',
+            'ceval-test-college_physics',
+            'ceval-test-college_chemistry',
+            'ceval-test-advanced_mathematics',
+            'ceval-test-probability_and_statistics',
+            'ceval-test-discrete_mathematics',
+            'ceval-test-electrical_engineer',
+            'ceval-test-metrology_engineer',
+            'ceval-test-high_school_mathematics',
+            'ceval-test-high_school_physics',
+            'ceval-test-high_school_chemistry',
+            'ceval-test-high_school_biology',
+            'ceval-test-middle_school_mathematics',
+            'ceval-test-middle_school_biology',
+            'ceval-test-middle_school_physics',
+            'ceval-test-middle_school_chemistry',
+            'ceval-test-veterinary_medicine',
+            'ceval-test-college_economics',
+            'ceval-test-business_administration',
+            'ceval-test-marxism',
+            'ceval-test-mao_zedong_thought',
+            'ceval-test-education_science',
+            'ceval-test-teacher_qualification',
+            'ceval-test-high_school_politics',
+            'ceval-test-high_school_geography',
+            'ceval-test-middle_school_politics',
+            'ceval-test-middle_school_geography',
+            'ceval-test-modern_chinese_history',
+            'ceval-test-ideological_and_moral_cultivation',
+            'ceval-test-logic',
+            'ceval-test-law',
+            'ceval-test-chinese_language_and_literature',
+            'ceval-test-art_studies',
+            'ceval-test-professional_tour_guide',
+            'ceval-test-legal_professional',
+            'ceval-test-high_school_chinese',
+            'ceval-test-high_school_history',
+            'ceval-test-middle_school_history',
+            'ceval-test-civil_servant',
+            'ceval-test-sports_science',
+            'ceval-test-plant_protection',
+            'ceval-test-basic_medicine',
+            'ceval-test-clinical_medicine',
+            'ceval-test-urban_and_rural_planner',
+            'ceval-test-accountant',
+            'ceval-test-fire_engineer',
+            'ceval-test-environmental_impact_assessment_engineer',
+            'ceval-test-tax_accountant',
+            'ceval-test-physician',
+        ],
+        weights=dict({
+            'ceval-test-accountant': 443,
+            'ceval-test-advanced_mathematics': 173,
+            'ceval-test-art_studies': 298,
+            'ceval-test-basic_medicine': 175,
+            'ceval-test-business_administration': 301,
+            'ceval-test-chinese_language_and_literature': 209,
+            'ceval-test-civil_servant': 429,
+            'ceval-test-clinical_medicine': 200,
+            'ceval-test-college_chemistry': 224,
+            'ceval-test-college_economics': 497,
+            'ceval-test-college_physics': 176,
+            'ceval-test-college_programming': 342,
+            'ceval-test-computer_architecture': 193,
+            'ceval-test-computer_network': 171,
+            'ceval-test-discrete_mathematics': 153,
+            'ceval-test-education_science': 270,
+            'ceval-test-electrical_engineer': 339,
+            'ceval-test-environmental_impact_assessment_engineer': 281,
+            'ceval-test-fire_engineer': 282,
+            'ceval-test-high_school_biology': 175,
+            'ceval-test-high_school_chemistry': 172,
+            'ceval-test-high_school_chinese': 178,
+            'ceval-test-high_school_geography': 178,
+            'ceval-test-high_school_history': 182,
+            'ceval-test-high_school_mathematics': 166,
+            'ceval-test-high_school_physics': 175,
+            'ceval-test-high_school_politics': 176,
+            'ceval-test-ideological_and_moral_cultivation': 172,
+            'ceval-test-law': 221,
+            'ceval-test-legal_professional': 215,
+            'ceval-test-logic': 204,
+            'ceval-test-mao_zedong_thought': 219,
+            'ceval-test-marxism': 179,
+            'ceval-test-metrology_engineer': 219,
+            'ceval-test-middle_school_biology': 192,
+            'ceval-test-middle_school_chemistry': 185,
+            'ceval-test-middle_school_geography': 108,
+            'ceval-test-middle_school_history': 207,
+            'ceval-test-middle_school_mathematics': 177,
+            'ceval-test-middle_school_physics': 178,
+            'ceval-test-middle_school_politics': 193,
+            'ceval-test-modern_chinese_history': 212,
+            'ceval-test-operating_system': 179,
+            'ceval-test-physician': 443,
+            'ceval-test-plant_protection': 199,
+            'ceval-test-probability_and_statistics': 166,
+            'ceval-test-professional_tour_guide': 266,
+            'ceval-test-sports_science': 180,
+            'ceval-test-tax_accountant': 443,
+            'ceval-test-teacher_qualification': 399,
+            'ceval-test-urban_and_rural_planner': 418,
+            'ceval-test-veterinary_medicine': 210
+        })),
+    dict(
+        name='bbh',
+        subsets=[
+            'bbh-temporal_sequences',
+            'bbh-disambiguation_qa',
+            'bbh-date_understanding',
+            'bbh-tracking_shuffled_objects_three_objects',
+            'bbh-penguins_in_a_table',
+            'bbh-geometric_shapes',
+            'bbh-snarks',
+            'bbh-ruin_names',
+            'bbh-tracking_shuffled_objects_seven_objects',
+            'bbh-tracking_shuffled_objects_five_objects',
+            'bbh-logical_deduction_three_objects',
+            'bbh-hyperbaton',
+            'bbh-logical_deduction_five_objects',
+            'bbh-logical_deduction_seven_objects',
+            'bbh-movie_recommendation',
+            'bbh-salient_translation_error_detection',
+            'bbh-reasoning_about_colored_objects',
+            'bbh-multistep_arithmetic_two',
+            'bbh-navigate',
+            'bbh-dyck_languages',
+            'bbh-word_sorting',
+            'bbh-sports_understanding',
+            'bbh-boolean_expressions',
+            'bbh-object_counting',
+            'bbh-formal_fallacies',
+            'bbh-causal_judgement',
+            'bbh-web_of_lies',
+        ]),
+    dict(
+        name='GaokaoBench',
+        subsets=[
+            'GaokaoBench_2010-2022_Math_II_MCQs',
+            'GaokaoBench_2010-2022_Math_I_MCQs',
+            'GaokaoBench_2010-2022_History_MCQs',
+            'GaokaoBench_2010-2022_Biology_MCQs',
+            'GaokaoBench_2010-2022_Political_Science_MCQs',
+            'GaokaoBench_2010-2022_Physics_MCQs',
+            'GaokaoBench_2010-2022_Chemistry_MCQs',
+            'GaokaoBench_2010-2013_English_MCQs',
+            'GaokaoBench_2010-2022_Chinese_Modern_Lit',
+            'GaokaoBench_2010-2022_English_Fill_in_Blanks',
+            'GaokaoBench_2012-2022_English_Cloze_Test',
+            'GaokaoBench_2010-2022_Geography_MCQs',
+            'GaokaoBench_2010-2022_English_Reading_Comp',
+            'GaokaoBench_2010-2022_Chinese_Lang_and_Usage_MCQs',
+        ],
+        weights=dict({
+            'GaokaoBench_2010-2013_English_MCQs': 105,
+            'GaokaoBench_2010-2022_Biology_MCQs': 900,
+            'GaokaoBench_2010-2022_Chemistry_MCQs': 744,
+            'GaokaoBench_2010-2022_Chinese_Lang_and_Usage_MCQs': 240,
+            'GaokaoBench_2010-2022_Chinese_Modern_Lit': 261,
+            'GaokaoBench_2010-2022_English_Fill_in_Blanks': 900.0,
+            'GaokaoBench_2010-2022_English_Reading_Comp': 940,
+            'GaokaoBench_2010-2022_Geography_MCQs': 380,
+            'GaokaoBench_2010-2022_History_MCQs': 1148,
+            'GaokaoBench_2010-2022_Math_II_MCQs': 1090,
+            'GaokaoBench_2010-2022_Math_I_MCQs': 1070,
+            'GaokaoBench_2010-2022_Physics_MCQs': 384,
+            'GaokaoBench_2010-2022_Political_Science_MCQs': 1280,
+            'GaokaoBench_2012-2022_English_Cloze_Test': 260
+        })),
+    dict(
+        name='flores_100_Indo-European-Germanic_English',
+        subsets=[
+            'flores_100_afr-eng',
+            'flores_100_dan-eng',
+            'flores_100_deu-eng',
+            'flores_100_isl-eng',
+            'flores_100_ltz-eng',
+            'flores_100_nld-eng',
+            'flores_100_nob-eng',
+            'flores_100_swe-eng',
+        ]),
+    dict(
+        name='flores_100_English_Indo-European-Germanic',
+        subsets=[
+            'flores_100_eng-afr',
+            'flores_100_eng-dan',
+            'flores_100_eng-deu',
+            'flores_100_eng-isl',
+            'flores_100_eng-ltz',
+            'flores_100_eng-nld',
+            'flores_100_eng-nob',
+            'flores_100_eng-swe',
+        ]),
+    dict(
+        name='flores_100_Indo-European-Romance_English',
+        subsets=[
+            'flores_100_ast-eng',
+            'flores_100_cat-eng',
+            'flores_100_fra-eng',
+            'flores_100_glg-eng',
+            'flores_100_oci-eng',
+            'flores_100_por-eng',
+            'flores_100_ron-eng',
+            'flores_100_spa-eng',
+        ]),
+    dict(
+        name='flores_100_English_Indo-European-Romance',
+        subsets=[
+            'flores_100_eng-ast',
+            'flores_100_eng-cat',
+            'flores_100_eng-fra',
+            'flores_100_eng-glg',
+            'flores_100_eng-oci',
+            'flores_100_eng-por',
+            'flores_100_eng-ron',
+            'flores_100_eng-spa',
+        ]),
+    dict(
+        name='flores_100_Indo-European-Slavic_English',
+        subsets=[
+            'flores_100_bel-eng',
+            'flores_100_bos-eng',
+            'flores_100_bul-eng',
+            'flores_100_ces-eng',
+            'flores_100_hrv-eng',
+            'flores_100_mkd-eng',
+            'flores_100_pol-eng',
+            'flores_100_rus-eng',
+            'flores_100_slk-eng',
+            'flores_100_slv-eng',
+            'flores_100_srp-eng',
+            'flores_100_ukr-eng',
+        ]),
+    dict(
+        name='flores_100_English_Indo-European-Slavic',
+        subsets=[
+            'flores_100_eng-bel',
+            'flores_100_eng-bos',
+            'flores_100_eng-bul',
+            'flores_100_eng-ces',
+            'flores_100_eng-hrv',
+            'flores_100_eng-mkd',
+            'flores_100_eng-pol',
+            'flores_100_eng-rus',
+            'flores_100_eng-slk',
+            'flores_100_eng-slv',
+            'flores_100_eng-srp',
+            'flores_100_eng-ukr',
+        ]),
+    dict(
+        name='flores_100_Indo-European-Indo-Aryan_English',
+        subsets=[
+            'flores_100_asm-eng',
+            'flores_100_ben-eng',
+            'flores_100_guj-eng',
+            'flores_100_hin-eng',
+            'flores_100_mar-eng',
+            'flores_100_npi-eng',
+            'flores_100_ory-eng',
+            'flores_100_pan-eng',
+            'flores_100_snd-eng',
+            'flores_100_urd-eng',
+        ]),
+    dict(
+        name='flores_100_English_Indo-European-Indo-Aryan',
+        subsets=[
+            'flores_100_eng-asm',
+            'flores_100_eng-ben',
+            'flores_100_eng-guj',
+            'flores_100_eng-hin',
+            'flores_100_eng-mar',
+            'flores_100_eng-npi',
+            'flores_100_eng-ory',
+            'flores_100_eng-pan',
+            'flores_100_eng-snd',
+            'flores_100_eng-urd',
+        ]),
+    dict(
+        name='flores_100_Indo-European-Other_English',
+        subsets=[
+            'flores_100_ckb-eng',
+            'flores_100_cym-eng',
+            'flores_100_ell-eng',
+            'flores_100_fas-eng',
+            'flores_100_gle-eng',
+            'flores_100_hye-eng',
+            'flores_100_ita-eng',
+            'flores_100_lav-eng',
+            'flores_100_lit-eng',
+            'flores_100_pus-eng',
+            'flores_100_tgk-eng',
+        ]),
+    dict(
+        name='flores_100_English_Indo-European-Other',
+        subsets=[
+            'flores_100_eng-ckb',
+            'flores_100_eng-cym',
+            'flores_100_eng-ell',
+            'flores_100_eng-fas',
+            'flores_100_eng-gle',
+            'flores_100_eng-hye',
+            'flores_100_eng-ita',
+            'flores_100_eng-lav',
+            'flores_100_eng-lit',
+            'flores_100_eng-pus',
+            'flores_100_eng-tgk',
+        ]),
+    dict(
+        name='flores_100_Austronesian_English',
+        subsets=[
+            'flores_100_ceb-eng',
+            'flores_100_ind-eng',
+            'flores_100_jav-eng',
+            'flores_100_mri-eng',
+            'flores_100_msa-eng',
+            'flores_100_tgl-eng',
+        ]),
+    dict(
+        name='flores_100_English_Austronesian',
+        subsets=[
+            'flores_100_eng-ceb',
+            'flores_100_eng-ind',
+            'flores_100_eng-jav',
+            'flores_100_eng-mri',
+            'flores_100_eng-msa',
+            'flores_100_eng-tgl',
+        ]),
+    dict(
+        name='flores_100_Atlantic-Congo_English',
+        subsets=[
+            'flores_100_ibo-eng',
+            'flores_100_kam-eng',
+            'flores_100_kea-eng',
+            'flores_100_lin-eng',
+            'flores_100_lug-eng',
+            'flores_100_nso-eng',
+            'flores_100_nya-eng',
+            'flores_100_sna-eng',
+            'flores_100_swh-eng',
+            'flores_100_umb-eng',
+            'flores_100_wol-eng',
+            'flores_100_xho-eng',
+            'flores_100_yor-eng',
+            'flores_100_zul-eng',
+        ]),
+    dict(
+        name='flores_100_English_Atlantic-Congo',
+        subsets=[
+            'flores_100_eng-ibo',
+            'flores_100_eng-kam',
+            'flores_100_eng-kea',
+            'flores_100_eng-lin',
+            'flores_100_eng-lug',
+            'flores_100_eng-nso',
+            'flores_100_eng-nya',
+            'flores_100_eng-sna',
+            'flores_100_eng-swh',
+            'flores_100_eng-umb',
+            'flores_100_eng-wol',
+            'flores_100_eng-xho',
+            'flores_100_eng-yor',
+            'flores_100_eng-zul',
+        ]),
+    dict(
+        name='flores_100_Afro-Asiatic_English',
+        subsets=[
+            'flores_100_amh-eng',
+            'flores_100_ara-eng',
+            'flores_100_ful-eng',
+            'flores_100_mlt-eng',
+            'flores_100_orm-eng',
+            'flores_100_som-eng',
+        ]),
+    dict(
+        name='flores_100_English_Afro-Asiatic',
+        subsets=[
+            'flores_100_eng-amh',
+            'flores_100_eng-ara',
+            'flores_100_eng-ful',
+            'flores_100_eng-mlt',
+            'flores_100_eng-orm',
+            'flores_100_eng-som',
+        ]),
+    dict(
+        name='flores_100_Turkic_English',
+        subsets=[
+            'flores_100_azj-eng',
+            'flores_100_kaz-eng',
+            'flores_100_kir-eng',
+            'flores_100_tur-eng',
+            'flores_100_uzb-eng',
+        ]),
+    dict(
+        name='flores_100_English_Turkic',
+        subsets=[
+            'flores_100_eng-azj',
+            'flores_100_eng-kaz',
+            'flores_100_eng-kir',
+            'flores_100_eng-tur',
+            'flores_100_eng-uzb',
+        ]),
+    dict(
+        name='flores_100_Dravidian_English',
+        subsets=[
+            'flores_100_kan-eng',
+            'flores_100_mal-eng',
+            'flores_100_tam-eng',
+            'flores_100_tel-eng',
+        ]),
+    dict(
+        name='flores_100_English_Dravidian',
+        subsets=[
+            'flores_100_eng-kan',
+            'flores_100_eng-mal',
+            'flores_100_eng-tam',
+            'flores_100_eng-tel',
+        ]),
+    dict(
+        name='flores_100_Sino-Tibetan_English',
+        subsets=[
+            'flores_100_mya-eng',
+            'flores_100_zho_simpl-eng',
+            'flores_100_zho_trad-eng',
+        ]),
+    dict(
+        name='flores_100_English_Sino-Tibetan',
+        subsets=[
+            'flores_100_eng-mya',
+            'flores_100_eng-zho_simpl',
+            'flores_100_eng-zho_trad',
+        ]),
+    dict(
+        name='flores_100_Other_English',
+        subsets=[
+            'flores_100_est-eng',
+            'flores_100_fin-eng',
+            'flores_100_hau-eng',
+            'flores_100_heb-eng',
+            'flores_100_hun-eng',
+            'flores_100_jpn-eng',
+            'flores_100_kat-eng',
+            'flores_100_khm-eng',
+            'flores_100_kor-eng',
+            'flores_100_lao-eng',
+            'flores_100_luo-eng',
+            'flores_100_mon-eng',
+            'flores_100_tha-eng',
+            'flores_100_vie-eng',
+        ]),
+    dict(
+        name='flores_100_English_Other',
+        subsets=[
+            'flores_100_eng-est',
+            'flores_100_eng-fin',
+            'flores_100_eng-hau',
+            'flores_100_eng-heb',
+            'flores_100_eng-hun',
+            'flores_100_eng-jpn',
+            'flores_100_eng-kat',
+            'flores_100_eng-khm',
+            'flores_100_eng-kor',
+            'flores_100_eng-lao',
+            'flores_100_eng-luo',
+            'flores_100_eng-mon',
+            'flores_100_eng-tha',
+            'flores_100_eng-vie',
+        ]),
+    dict(
+        name='flores_100',
+        subsets=[
+            'flores_100_afr-eng',
+            'flores_100_dan-eng',
+            'flores_100_deu-eng',
+            'flores_100_isl-eng',
+            'flores_100_ltz-eng',
+            'flores_100_nld-eng',
+            'flores_100_nob-eng',
+            'flores_100_swe-eng',
+            'flores_100_ast-eng',
+            'flores_100_cat-eng',
+            'flores_100_fra-eng',
+            'flores_100_glg-eng',
+            'flores_100_oci-eng',
+            'flores_100_por-eng',
+            'flores_100_ron-eng',
+            'flores_100_spa-eng',
+            'flores_100_bel-eng',
+            'flores_100_bos-eng',
+            'flores_100_bul-eng',
+            'flores_100_ces-eng',
+            'flores_100_hrv-eng',
+            'flores_100_mkd-eng',
+            'flores_100_pol-eng',
+            'flores_100_rus-eng',
+            'flores_100_slk-eng',
+            'flores_100_slv-eng',
+            'flores_100_srp-eng',
+            'flores_100_ukr-eng',
+            'flores_100_asm-eng',
+            'flores_100_ben-eng',
+            'flores_100_guj-eng',
+            'flores_100_hin-eng',
+            'flores_100_mar-eng',
+            'flores_100_npi-eng',
+            'flores_100_ory-eng',
+            'flores_100_pan-eng',
+            'flores_100_snd-eng',
+            'flores_100_urd-eng',
+            'flores_100_ckb-eng',
+            'flores_100_cym-eng',
+            'flores_100_ell-eng',
+            'flores_100_fas-eng',
+            'flores_100_gle-eng',
+            'flores_100_hye-eng',
+            'flores_100_ita-eng',
+            'flores_100_lav-eng',
+            'flores_100_lit-eng',
+            'flores_100_pus-eng',
+            'flores_100_tgk-eng',
+            'flores_100_ceb-eng',
+            'flores_100_ind-eng',
+            'flores_100_jav-eng',
+            'flores_100_mri-eng',
+            'flores_100_msa-eng',
+            'flores_100_tgl-eng',
+            'flores_100_ibo-eng',
+            'flores_100_kam-eng',
+            'flores_100_kea-eng',
+            'flores_100_lin-eng',
+            'flores_100_lug-eng',
+            'flores_100_nso-eng',
+            'flores_100_nya-eng',
+            'flores_100_sna-eng',
+            'flores_100_swh-eng',
+            'flores_100_umb-eng',
+            'flores_100_wol-eng',
+            'flores_100_xho-eng',
+            'flores_100_yor-eng',
+            'flores_100_zul-eng',
+            'flores_100_amh-eng',
+            'flores_100_ara-eng',
+            'flores_100_ful-eng',
+            'flores_100_mlt-eng',
+            'flores_100_orm-eng',
+            'flores_100_som-eng',
+            'flores_100_azj-eng',
+            'flores_100_kaz-eng',
+            'flores_100_kir-eng',
+            'flores_100_tur-eng',
+            'flores_100_uzb-eng',
+            'flores_100_kan-eng',
+            'flores_100_mal-eng',
+            'flores_100_tam-eng',
+            'flores_100_tel-eng',
+            'flores_100_mya-eng',
+            'flores_100_zho_simpl-eng',
+            'flores_100_zho_trad-eng',
+            'flores_100_est-eng',
+            'flores_100_fin-eng',
+            'flores_100_hau-eng',
+            'flores_100_heb-eng',
+            'flores_100_hun-eng',
+            'flores_100_jpn-eng',
+            'flores_100_kat-eng',
+            'flores_100_khm-eng',
+            'flores_100_kor-eng',
+            'flores_100_lao-eng',
+            'flores_100_luo-eng',
+            'flores_100_mon-eng',
+            'flores_100_tha-eng',
+            'flores_100_vie-eng',
+            'flores_100_eng-afr',
+            'flores_100_eng-dan',
+            'flores_100_eng-deu',
+            'flores_100_eng-isl',
+            'flores_100_eng-ltz',
+            'flores_100_eng-nld',
+            'flores_100_eng-nob',
+            'flores_100_eng-swe',
+            'flores_100_eng-ast',
+            'flores_100_eng-cat',
+            'flores_100_eng-fra',
+            'flores_100_eng-glg',
+            'flores_100_eng-oci',
+            'flores_100_eng-por',
+            'flores_100_eng-ron',
+            'flores_100_eng-spa',
+            'flores_100_eng-bel',
+            'flores_100_eng-bos',
+            'flores_100_eng-bul',
+            'flores_100_eng-ces',
+            'flores_100_eng-hrv',
+            'flores_100_eng-mkd',
+            'flores_100_eng-pol',
+            'flores_100_eng-rus',
+            'flores_100_eng-slk',
+            'flores_100_eng-slv',
+            'flores_100_eng-srp',
+            'flores_100_eng-ukr',
+            'flores_100_eng-asm',
+            'flores_100_eng-ben',
+            'flores_100_eng-guj',
+            'flores_100_eng-hin',
+            'flores_100_eng-mar',
+            'flores_100_eng-npi',
+            'flores_100_eng-ory',
+            'flores_100_eng-pan',
+            'flores_100_eng-snd',
+            'flores_100_eng-urd',
+            'flores_100_eng-ckb',
+            'flores_100_eng-cym',
+            'flores_100_eng-ell',
+            'flores_100_eng-fas',
+            'flores_100_eng-gle',
+            'flores_100_eng-hye',
+            'flores_100_eng-ita',
+            'flores_100_eng-lav',
+            'flores_100_eng-lit',
+            'flores_100_eng-pus',
+            'flores_100_eng-tgk',
+            'flores_100_eng-ceb',
+            'flores_100_eng-ind',
+            'flores_100_eng-jav',
+            'flores_100_eng-mri',
+            'flores_100_eng-msa',
+            'flores_100_eng-tgl',
+            'flores_100_eng-ibo',
+            'flores_100_eng-kam',
+            'flores_100_eng-kea',
+            'flores_100_eng-lin',
+            'flores_100_eng-lug',
+            'flores_100_eng-nso',
+            'flores_100_eng-nya',
+            'flores_100_eng-sna',
+            'flores_100_eng-swh',
+            'flores_100_eng-umb',
+            'flores_100_eng-wol',
+            'flores_100_eng-xho',
+            'flores_100_eng-yor',
+            'flores_100_eng-zul',
+            'flores_100_eng-amh',
+            'flores_100_eng-ara',
+            'flores_100_eng-ful',
+            'flores_100_eng-mlt',
+            'flores_100_eng-orm',
+            'flores_100_eng-som',
+            'flores_100_eng-azj',
+            'flores_100_eng-kaz',
+            'flores_100_eng-kir',
+            'flores_100_eng-tur',
+            'flores_100_eng-uzb',
+            'flores_100_eng-kan',
+            'flores_100_eng-mal',
+            'flores_100_eng-tam',
+            'flores_100_eng-tel',
+            'flores_100_eng-mya',
+            'flores_100_eng-zho_simpl',
+            'flores_100_eng-zho_trad',
+            'flores_100_eng-est',
+            'flores_100_eng-fin',
+            'flores_100_eng-hau',
+            'flores_100_eng-heb',
+            'flores_100_eng-hun',
+            'flores_100_eng-jpn',
+            'flores_100_eng-kat',
+            'flores_100_eng-khm',
+            'flores_100_eng-kor',
+            'flores_100_eng-lao',
+            'flores_100_eng-luo',
+            'flores_100_eng-mon',
+            'flores_100_eng-tha',
+            'flores_100_eng-vie',
+        ]),
+    dict(
+        name='tydiqa-goldp',
+        subsets=[
+            'tydiqa-goldp_arabic',
+            'tydiqa-goldp_bengali',
+            'tydiqa-goldp_english',
+            'tydiqa-goldp_finnish',
+            'tydiqa-goldp_indonesian',
+            'tydiqa-goldp_japanese',
+            'tydiqa-goldp_korean',
+            'tydiqa-goldp_russian',
+            'tydiqa-goldp_swahili',
+            'tydiqa-goldp_telugu',
+            'tydiqa-goldp_thai',
+        ]),
+    dict(
+        name='xiezhi',
+        subsets=[
+            'xiezhi-spec_eng',
+            'xiezhi-spec_chn',
+            'xiezhi-inter_eng',
+            'xiezhi-inter_chn',
+        ]),
+    dict(
+        name='scibench',
+        subsets=[
+            'scibench-atkins',
+            'scibench-calculus',
+            'scibench-chemmc',
+            'scibench-class',
+            'scibench-diff',
+            'scibench-fund',
+            'scibench-matter',
+            'scibench-quan',
+            'scibench-stat',
+            'scibench-thermo',
+        ]),
+    dict(
+        name='scibench_zs-cot',
+        subsets=[
+            'scibench-atkins_zs-cot',
+            'scibench-calculus_zs-cot',
+            'scibench-chemmc_zs-cot',
+            'scibench-class_zs-cot',
+            'scibench-diff_zs-cot',
+            'scibench-fund_zs-cot',
+            'scibench-matter_zs-cot',
+            'scibench-quan_zs-cot',
+            'scibench-stat_zs-cot',
+            'scibench-thermo_zs-cot',
+        ]),
+    dict(
+        name='scibench_fs',
+        subsets=[
+            'scibench-atkins_fs',
+            'scibench-calculus_fs',
+            'scibench-chemmc_fs',
+            'scibench-class_fs',
+            'scibench-diff_fs',
+            'scibench-fund_fs',
+            'scibench-matter_fs',
+            'scibench-quan_fs',
+            'scibench-stat_fs',
+            'scibench-thermo_fs',
+        ]),
+    dict(
+        name='scibench_fs-cot',
+        subsets=[
+            'scibench-atkins_fs-cot',
+            'scibench-calculus_fs-cot',
+            'scibench-chemmc_fs-cot',
+            'scibench-class_fs-cot',
+            'scibench-diff_fs-cot',
+            'scibench-fund_fs-cot',
+            'scibench-matter_fs-cot',
+            'scibench-quan_fs-cot',
+            'scibench-stat_fs-cot',
+            'scibench-thermo_fs-cot',
+        ]),
+    dict(
+        name='mgsm_latin',
+        subsets=[
+            'mgsm_de',
+            'mgsm_en',
+            'mgsm_es',
+            'mgsm_fr',
+            'mgsm_sw',
+        ]),
+    dict(
+        name='mgsm_non_latin',
+        subsets=[
+            'mgsm_bn',
+            'mgsm_ja',
+            'mgsm_ru',
+            'mgsm_te',
+            'mgsm_th',
+            'mgsm_zh',
+        ]),
+    dict(
+        name='mgsm',
+        subsets=[
+            'mgsm_bn',
+            'mgsm_de',
+            'mgsm_en',
+            'mgsm_es',
+            'mgsm_fr',
+            'mgsm_ja',
+            'mgsm_ru',
+            'mgsm_sw',
+            'mgsm_te',
+            'mgsm_th',
+            'mgsm_zh',
+        ]),
+    dict(
+        name='longbench_single-document-qa',
+        subsets=[
+            'LongBench_narrativeqa',
+            'LongBench_qasper',
+            'LongBench_multifieldqa_en',
+            'LongBench_multifieldqa_zh',
+        ]),
+    dict(
+        name='longbench_multi-document-qa',
+        subsets=[
+            'LongBench_hotpotqa',
+            'LongBench_2wikimqa',
+            'LongBench_musique',
+            'LongBench_dureader',
+        ]),
+    dict(
+        name='longbench_summarization',
+        subsets=[
+            'LongBench_gov_report',
+            'LongBench_qmsum',
+            'LongBench_multi_news',
+            'LongBench_vcsum',
+        ]),
+    dict(
+        name='longbench_few-shot-learning',
+        subsets=[
+            'LongBench_trec',
+            'LongBench_triviaqa',
+            'LongBench_samsum',
+            'LongBench_lsht',
+        ]),
+    dict(
+        name='longbench_synthetic-tasks',
+        subsets=[
+            'LongBench_passage_count',
+            'LongBench_passage_retrieval_en',
+            'LongBench_passage_retrieval_zh',
+        ]),
+    dict(
+        name='longbench_code-completion',
+        subsets=[
+            'LongBench_lcc',
+            'LongBench_repobench-p',
+        ]),
+    dict(
+        name='longbench_zh',
+        subsets=[
+            'LongBench_multifieldqa_zh',
+            'LongBench_dureader',
+            'LongBench_vcsum',
+            'LongBench_lsht',
+            'LongBench_passage_retrieval_zh',
+            'LongBench_lcc',
+            'LongBench_repobench-p',
+        ]),
+    dict(
+        name='longbench_en',
+        subsets=[
+            'LongBench_narrativeqa',
+            'LongBench_qasper',
+            'LongBench_multifieldqa_en',
+            'LongBench_hotpotqa',
+            'LongBench_2wikimqa',
+            'LongBench_musique',
+            'LongBench_gov_report',
+            'LongBench_qmsum',
+            'LongBench_multi_news',
+            'LongBench_trec',
+            'LongBench_triviaqa',
+            'LongBench_samsum',
+            'LongBench_passage_count',
+            'LongBench_passage_retrieval_en',
+            'LongBench_lcc',
+            'LongBench_repobench-p',
+        ]),
+    dict(
+        name='longbench',
+        subsets=[
+            'longbench_single-document-qa',
+            'longbench_multi-document-qa',
+            'longbench_summarization',
+            'longbench_few-shot-learning',
+            'longbench_synthetic-tasks',
+            'longbench_code-completion',
+        ]),
+])
+work_dir = 'outputs/default/20250731_120452'

20250731_120452-unified-expand/summary/summary_20250731_120452.csv

@@ -0,0 +1,2 @@
+dataset,version,metric,mode,vllm-api-general-chat
+openai_humaneval,f4a973,humaneval_pass@1,gen,95.73

20250731_120452-unified-expand/summary/summary_20250731_120452.md

@@ -0,0 +1,3 @@
+| dataset | version | metric | mode | vllm-api-general-chat |
+|----- | ----- | ----- | ----- | -----|
+| openai_humaneval | f4a973 | humaneval_pass@1 | gen | 95.73 |

20250731_120452-unified-expand/summary/summary_20250731_120452.txt

@@ -0,0 +1,31 @@
+20250731_120452
+tabulate format
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+dataset           version    metric            mode      vllm-api-general-chat
+----------------  ---------  ----------------  ------  -----------------------
+openai_humaneval  f4a973     humaneval_pass@1  gen                       95.73
+$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$
+
+-------------------------------------------------------------------------------------------------------------------------------- THIS IS A DIVIDER --------------------------------------------------------------------------------------------------------------------------------
+
+csv format
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+dataset,version,metric,mode,vllm-api-general-chat
+openai_humaneval,f4a973,humaneval_pass@1,gen,95.73
+$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$
+
+markdown format
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+| dataset | version | metric | mode | vllm-api-general-chat |
+|----- | ----- | ----- | ----- | -----|
+| openai_humaneval | f4a973 | humaneval_pass@1 | gen | 95.73 |
+
+$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$
+-------------------------------------------------------------------------------------------------------------------------------- THIS IS A DIVIDER --------------------------------------------------------------------------------------------------------------------------------
+
+raw format
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+-------------------------------
+Model: vllm-api-general-chat
+openai_humaneval: {'humaneval_pass@1': 95.73170731707317}
+$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$