diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47bd89009fb74d4c468b6deec54322053c2c83c2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29925c347fe23503b67a4b1738cc8e909f81078a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..abe4ef94f0599c875caf5e5dec3b899042dd30e1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..751878fe1d333017230e639fda582d65ec304344 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55233f7f20811e1c95650dee1e92862a50b34f54 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a11e310c89164737d89c6281f9589c038c2635d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a31b4c268096dc79b15249de064268fec3d4d784 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22136569ebdb5755787b1bc37e7c74d94febf58c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18fc7a23e7470e44dc15fc47569c083aaecc0e6b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b3955a93e27cee34ab96ad17ab688eb22418043 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57095856fa16bed191f32241ee7b8485f7a770f5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed31f8cc97d0d35c0b6e31573c095aba1b847979 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07746d096513d1508cb8b113ef945f40e518f153 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3853e1622e915eed12b6d667fc82b644fa07f49a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b75ac9df71c2b002439dbfeac3e860c06692d07e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da8fd46ffe348f81607e2cf4d7072b0955ac0102 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ddd0ab3d121a4311a9eff4533460d1912a1f5239 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6be1ae817fc5f2d1127b05d31ab7b1cb3617a841 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cadc7f964caa20ddfa75e9ed057098a52203a6c0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72609ea90d020b5faf9fcea99139394e826ae330 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..319123c61992678604578547f13f95249b1fc62b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dec44c2938c6d3cec700c08e6cc2a0c22d5d1fa4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a18a3942d6a759c9a31529187f8c84bef3c30833 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b06431e21049838070abeb45458538d5d283d837 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d9d6b792ca9592b53f60e7a3b9fe52fcee62a42 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/global_mmlu_full_es_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/es/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_fa_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_fa_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff36e89df8f73d1e4fcd8f2817e572ea52007bbd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_fa_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: fa +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_global_mmlu_full_fa.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_global_mmlu_full_fa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..282664e59fff56be7b06eee62198b31a78f2899c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_global_mmlu_full_fa.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_fa +task: + - global_mmlu_full_fa_stem + - global_mmlu_full_fa_other + - global_mmlu_full_fa_social_sciences + - global_mmlu_full_fa_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f36ecea5f2b4ff98d998328d66b0087deeb33849 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fa_humanities +task: + - global_mmlu_full_fa_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd57bb86bb118224258068151bcfb77f91afa321 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fa_other +task: + - global_mmlu_full_fa_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e7da860f408f871f6d5f55ecf7bdd92c9aa81f8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fa_social_sciences +task: + - global_mmlu_full_fa_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bf2eb01368a6c86ce0fad30e04ee321dce8cc8f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fa_stem +task: + - global_mmlu_full_fa_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1014795f0ffabcb8cbd664a35c4dd1107276dc3c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..317705c92aa860af17b86d55acdddc4400d5b1a5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45475964110cab623ae40dc71f5e136e9db32fcf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c0dd60b702d06470110f6b94c8ef3ea7bbf31b8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7af0e21f2e84c0c38bd0839c95dd1e00f64f7b6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31ae6d71c4e6586b57c1f54fdcbd49c13644ba32 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b099f413a68244140b4ae16bb5d9231f4e34555 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07491e5bf7f39a26d409cf15d4613bb7cc537acf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..774f6b970e90239efc066e80b544dae0e88edda4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13d6f5a2ac910fcef23865e4215531105932408a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e415b8cf7782d520499d84f0101f4f77c6f3c13 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae47213b39e79bbee367dace28219e446e388a99 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3f2ba4ce1c465cbe343c6b358c241e8140d4b18 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6cf79a9229c102bd62100c56cd3d3404f7249026 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab7aa8584e93622225c370311282c518671fc21c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b83f6ddcf82ebbe572c7e76aa636332f10ff6efe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cab2effa03a184bab3d8486530151642708f6e75 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93d11b75b9a4039866238ef565766e69dbfb45eb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59b6869bd685cb7cabd763d03c3497437c267ddd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d15d4b4ee4336f73450e0415f949be6404050b2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a02df4f13acfff661358bcfccb76fea1d0a74adf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e18b2c7b6bfe10d4a3c0bc834c5e64c1ef4725d4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d94c7e89fb70e84099c6be16ddcb938e10a96e27 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e10078959c1e99b93684ed5f9cb17d58cec8d3fc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9ad06337b8e6cdfd3b474d9baf2e2ced4f267a6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c733b17def83597d8787fefd9a12df5ed63ba53 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f88f0aa335190798cdc01c9acd0927822c9065f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64fdef984a0f16d87645682bf15a4df59f19dddd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c43a115be4261a4ac303ef88a5b0b34471b1837b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebb4e82d7390d9f8496a5b31bb4b07ada19f87fa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0041e33e03b3ba765e14bc5d0e91eee76792382 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66f38f54bbbab488cb994b94e956bbd41772f1fe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95a2addea9f833168e40a19cdbedf741852e96a8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..475a71fdbbeb22532ef620ceda4bc70d03445948 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0d6aec2bbcf9c60bbb450ecb4e5fa4fec25a851 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d82bd5a64a031539f02338d163082f2c6bdc026 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8e89d3c7c16793541b93f5ad406a753e81059ea --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e4d1a8a7ad94e6ac37a4ca3530cb982e0f9bb50 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7e592bae4a02a3b7959964f49f3fc5a58e49c6b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0e7ef1f42c6ec8e42bc7bcb346af99c9f136cdf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c31679ec0ce09c4626045b59dcc0e835d3f28cc7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..652d5a3335974f6e32ab58371e8b6dce74d1c54b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16adcb26284ac93d5fcc64cffad556399bd35514 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92d018f2508759a5eba64ab2a17867321e7e43f4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae7e065ecafdad36c7fc421fb05146ef3a8ffee3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd8513da780e3fbc9f9ea604c7b3b4a6800e5bd9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9fd6bb3d0a359bef99ff00477d3bb70c0b889bcd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99f6c316413309444d15268e4925aabed113e9b7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9fee460a41ec892865ec1ebc540d91fb4d219abb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13d67d457680de3c7935f0b4b4c2571378a65201 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e821145e4170de1337027c30b58aa9bc833e443 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de6cc3118d20e7ef0a8ae248466192ed03134eef --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64d5fd14a5ae149470df306e0b997372ae863935 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf3d9564b1281dd85210e9a08eca428f1cd96bed --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38d51936c799103492b48f79c1f9effd5d4b0cb8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39c5188d3406d388333cd93ed08a107c78f9d5e4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..44e6fc82993da5e4543092433e2b13868281b765 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/global_mmlu_full_fa_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fa/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_fil_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_fil_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..b65864138d21515e21bb5b5e813fbce4f76ee904 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_fil_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: fil +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_global_mmlu_full_fil.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_global_mmlu_full_fil.yaml new file mode 100644 index 0000000000000000000000000000000000000000..659c9d4b9a60b23818fd4b4e6617f9df92ded70d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_global_mmlu_full_fil.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_fil +task: + - global_mmlu_full_fil_stem + - global_mmlu_full_fil_other + - global_mmlu_full_fil_social_sciences + - global_mmlu_full_fil_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..061eb818918e2279668ecc6087e8f76de9a6698c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fil_humanities +task: + - global_mmlu_full_fil_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fea793eff0e83703853feb8b28d5f0f4a24dda7b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fil_other +task: + - global_mmlu_full_fil_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9f793305951ecd337a7522f986bc5a5935f5a16 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fil_social_sciences +task: + - global_mmlu_full_fil_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e567c70b16de81204a0bca56062d14c4a5fe2da --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fil_stem +task: + - global_mmlu_full_fil_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7eef19d6d1a0c959e3d7bdf60adbc8e02f8e0a48 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e87d8d803487fb44ddd992d57b13bbdd82e54e7e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c258877fca7f68475467dc32a530d0274abc841 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..139f3cccffb7e55ae7128ed615b76c561aa1fc01 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc160a99f142e662840d33b82d51c867a4068ef5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff6fa3d845e14df911bcdd5ad23da3e63c7588d0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61f0df50865a6b95a9237e6ff1bc8ada97660c8d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1385b9345d53f99b935764d70b4a63d31bec5af3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afe15d7d57504b28c4dcf4980b0db8d9971612f7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..221289f7d0ec8a5c1d31f0f12f0f688a00fe9195 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..863792b3cf27e931b4d42c0f96f1f3c68267a919 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7971c6065cc62f24e9672417f4bdb812092aa7aa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77a75ccf0c4bd02d58b1d9c2a9620e9fcf5e923e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd98fc8db07bb15298c0e7c6bdd82b9c184bd408 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98e48a27cc8962d5bbafcd9a39ed9044b2a374ba --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eba4149c2e842f4c9902f2052d9cfe47b041d751 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1796059f72356dd890f53bfd842957e38aaa90e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96886181c13c2379a0a13d412a981158dc367e2a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93d94120d327f185b11aefc04e58e9e68391c8bd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ec56d5bee1b08d299e5bb5f28acc3d921cca59e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82d86aed2344f5f4e1e19231769a5ebeddf38fae --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7fcdec0a10b90d8fd61f9a2089f6d3f048e2dc76 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96268192265344046079f9e7abdfb9a40a02bfee --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f826de3daa160c9ced3f383fda400da34c0b4930 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..104a70881b304fd59ead25e02893891851fa8359 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d499b4d948b2a988af49faade7a275450730dd0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43fcc04d93cd66edeae07a40d69e0497384e5695 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..175f31eef74c214fd11b660c4cff66ae8196e128 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fc2dd5ca57af8a8c3df6d79cf809380cfbacf26 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0540d57c845a41600aa31326745490af328374ad --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0801af2d0536d6959034c65bd6156e4cd4e088e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..724b7ce81de1764f0b0ba878fd6d93d408941cb7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c2c114123967f126ca40f7a22cf5eca97c3a743 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1672d5b2663ad38baf140903aeece3b5ad87db64 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c5da91c50cc864d0068464ac52c4ad851e3a9ce --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dea2b20b8f28d320e73a02b23af2736bf3ace1f0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a30c7243f07dd46188b08085c8440724a9da418 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d2a7062c232c0165020b3cdd7d9788950fb5a0a9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ea5683556e38fc3eb03128709d99fbef2e12cd4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82d4490adda68f3db5cb97bf3514d5ac7868d2d9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bdeb0984f5d5e8b5a18c38bf34a6bfdf67321152 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51c56a3bee09eda53a834b3f712d4ddadefa35e7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53148a54249fdfd2104cf5f92357f151fb25b3c1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb5fecf2edf0d8d6625d9f27b33a769ff45ef51d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35859dc2da609cf6fd8e498749e747e638d3c459 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc2d414eb1c67c48da44275f4a0b189bf6ee24b5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..abf65fd2fcb66180ed45032df7e65960bd21473b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04ce343642b8af545083b1dc21ad269f512d2c94 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5694cf45bb2db7ae2a0a2dcf60b2d9b696842bd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7fd0446f901869c9a1141455f9c3d55b56dcc37 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9ce14aa062dd5ce11253374373157c24d6160b4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fdae52986567539b429436b2ff5609a30856307c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a03eec195604894bf1666fbf745699a98f6734c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc0ed0523e6448b4aa339636c460741ee11d3e47 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed40afb61259b660310b1a4c7d889997754c6e34 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85ed4d427fb75031c99277423fe9765575d56eff --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ee6bce1834b1a20e3b1d23946e97ff1034cc5c7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/global_mmlu_full_fil_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fil/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_fr_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_fr_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a9a94acb119cbbee7ffd629386d12f19903a58d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_fr_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: fr +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_global_mmlu_full_fr.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_global_mmlu_full_fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7857a193281519c94d2b087e47ecb87d93ba7be9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_global_mmlu_full_fr.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_fr +task: + - global_mmlu_full_fr_stem + - global_mmlu_full_fr_other + - global_mmlu_full_fr_social_sciences + - global_mmlu_full_fr_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..697e3a29e03b0a03635f6ba49905682ddbd4698d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fr_humanities +task: + - global_mmlu_full_fr_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b2ada6b32d371f79cedc9539c3ea21a9bb68e79 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fr_other +task: + - global_mmlu_full_fr_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac7e4605614a20ccd67d8d88172436c105a8098f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fr_social_sciences +task: + - global_mmlu_full_fr_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c81d601f35758afefaa6b804981941f494b8196e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fr_stem +task: + - global_mmlu_full_fr_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf7d76c383a6d8087c010b86e8fe7c5d2883d6b7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9a96927c00636d01609cb2564dc5d3fee228638 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e4ca5a7691cd4800a24ff7032e1fa66106bdc72 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df3c1fbd6b78ff26410002dfdd525d69d0a6c876 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0daa2e6ee66095c0065e05c3f5f966cb042e9ca --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e997578893b0bd98fff5ebb2696e4e759a032ca --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c1c31897fcab74c2eefd8e564c04f5519230a7e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..078108f8c7e73ea0a2bd731648fac024b5d5a042 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf2f2940e1ac269c5676cadc3639bd720cae70fc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c9ccc80e79accc520c4c865f090ea399897aa40 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01dcea37091e85c421f9c4a3009d5159f69f5ba5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..794f64be1db599f4bd21d60297397901463e6bbf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12c6afc2abd8fb8728d727455f77bd7e77b49c72 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f33ddaba54867b2d651a3896b1d87562efae508 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dcb058564a29b77f441e872340594c0f8b42d56 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2658ce96a477c39d7a8fa6f166b73aa0a4bf0797 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5239cb1c194a08fb6adf2b6535ae5e2fcbfb6776 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2763dcb53d53033bf7f6389f0b64d2f361def3e7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a6a26c9bf4bc956d29fc0c6d7cad66e2b2cbe80 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ffacc2982af8fce14dca0d277e2795563dbf966 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d17204224942f109607411c8d47fd1d23148419f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9788e7be702d49a032954c193524636049a1ed39 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e2ff22eae27b956947253cc1200bc2ffaba2bc8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2aba3b6121ec8f3075e1d3873c417c8003b5a962 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21fb1df5b34f62551946632c754c03e6b86ea1ce --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a975d1fcc7cde59838f4e25530e5b4e075a7d455 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff654ff35a003c1c2392d78437c25bfcf28ce772 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4038c956dbeeee5668641b0ae4204d1a91bff2eb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a65da78026eadd2ceeae2461884bbec1156f24af --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37c75136b2ba62f3ddf2170ef5db426ebf4ddb48 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0e123f267894898a64f0cb16ca6e02f4c5e38f3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2a9cf6b143899e697e80d2f5cfd0316f6727c5c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9e9ece96b7b55d26517e842e719ef28400cab36 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eac30d27f9147f22c136b90fdf54f6235b750033 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e15b0fb1064d77e6035e825a34ef13c9162e900 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42079c92eb665517e9afaeabbc016d5ae2425fd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68ebdb717525e90095a2284aa9d0fc95c2018676 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25a8df3a87ed22e928ece07116306dc3b488f48b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73f7d869215c82216e9f3826f1313a25d2834ec0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a19b83e9a2855f0dadfcd8c5c33d962ab5733d9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d15774f13f7766baa9f4a77a2e11145c75fc1b99 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7519709782e1dd04c51f87a91ef32f403ae4e2c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f625921e5d4ebd4136ee0e36d6725a9e856ebf45 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4575ca044297eff4da62096a1f4911349c559940 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61521c91da4d5597aa1c31450e1b18a0115f9771 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f771c34254a811960b476324175d5592414c237 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3bcac0f5b900f65de1b502464da1275bf7bb261c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a06a7af518877583fe9f1e1a83eac47afa9f54cd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ecf2e8e7014784d6a6f644894d719b6a36f3df0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..983a2d38053e4beeb5539064de4583e3c9f59017 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59d9aa309b3c175f590251de47f138b34feea201 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d84b7ad07cee145f31c5ec81f7a3abb02abb089c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fcd82b7d8d6e088d3b5aed15dbb708891bf75aa4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff7b8fd3185d377be099f72e04ba03e21d4839eb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d92c20955a118004738b6a7c606dc72a0e7115ad --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..211c96a79fe7cf5166cd18d10e538c9317537646 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1f168aed5d7ba6d62ab40da70f71b00916bb620 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/global_mmlu_full_fr_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/fr/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_global_mmlu_full_ha.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_global_mmlu_full_ha.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a50084174aebe37f832c02ca23f5a2fa7c1b9f09 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_global_mmlu_full_ha.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_ha +task: + - global_mmlu_full_ha_stem + - global_mmlu_full_ha_other + - global_mmlu_full_ha_social_sciences + - global_mmlu_full_ha_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84cce38d710ab046922d3b371b68f9b939f4e248 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ha_humanities +task: + - global_mmlu_full_ha_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73a6ea0dea1b57902569321973002fbfc072453c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ha_other +task: + - global_mmlu_full_ha_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b520a5dc50a793f2cf5e349baf416b69cf76332 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ha_social_sciences +task: + - global_mmlu_full_ha_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6213d280b995bbc25881afcc989969965cce1308 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ha_stem +task: + - global_mmlu_full_ha_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_ha_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_ha_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d0262a67b5c7621fd377dbc2e948675766f6acf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/_ha_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: ha +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62ad5e7d7bec45d82452d683a6a7682b65dce287 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ead0f6c11557f972900bbc21a4a94e578e73d93 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1616398f862e1ff6b94b6ee28a60596eb5c44110 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1719b0e0d609b3af0175267e071dc9bfa09d87e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcef5e272db17444b7ccb484be0cbca2d012e181 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f28256946943960f33f8f6b4bac63f62f755284c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73a422e3d790f5e70d6990f3008f6131f518aed6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7bdb65c9e0542257eeb7e804a5c33f864b2264f5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47e5326cea33d8427b45c5f1c5b7bff2bb455767 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9065f0855ba147b109db2e21f9934cc6f57c4a56 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40aa11c5d1f38e6aee1268951603d5fcbd705e3f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38d1e9c256f0aa791db1e945d697dd740d7f0f99 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7326514a452183cfcc603c31a8dcda1530229fcc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e865b6bc620b2fbbb79df705cde11f1a982a18c9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9457d1bb90ffc2d818a9ca10a762866d2c243c55 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e04fb1ff63dfc2d5980103c175d9709e60fd09ea --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03c9cbac0617afc136a3ab0a1e11d7d64974f3e2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db104be57b25f8efd7f4238ade9ad6f7d757a24a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..729fed2bef95ce20bc22510908d99174ec2865eb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13f5621b2fb49aa66933cb9c2a38f8e1f2010065 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1914e1fb86082927733adab77490ffd613c1f93f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa878b037d99406d08fc104b5099366c39eb6ed6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10a136746e32025589512aad8a7e9cdf208ffabe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eebac40961da29be7dd22d2da3895e7efb7cbf3c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a22ab84293fdcdf9991cbca6ec33c70094615d3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc681f90927f465b5e1d7e95d3c7e889d4f01248 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81bb343c37307ce9aaacaf4f55ae3b832a0c1753 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bc4cc4d9d2c487ced03d857086daf7a768a8052 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5d46e5ec0358ce24e3cc92b571d6608a5d8fb29 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4848cc314c8038896b463119b1b02e8bfd42378d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a22c79abaa7ba6e9cd575e6898d09debcf4ad5b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13882279817265b164fab2e49556770333e673d6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51ff436ba4e0fd6d30c9cccccc1d7efbc19f6f1a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a36fb8650e36b4a71a5c9d65cf676cc02be0fab --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1c9cc1c43e37f85d3fa9e9de402d925e4b9754d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bc1314ba04e21d323124278e20ae87c4c7b5170 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..259534b9ad217196da25ef9397e136bbcfbfce0b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c94a073b9ff6a08893f2bf41b38a690ef95cbb8a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..666d47202e2169eb8aea1b15f7407b1851b2c3da --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9528a1f371cb4df01d394dcbfce4bc1098a2e56c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92f0a408d8079a133586d8936b1fd0b0dd35b477 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc97a8dc9459ce344ff61bc5273c2fea5d152ccf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbcf96c22ddebe8754e274b9d94f3c94aceaea7f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa7b426699d3e43acc47f8e2774b0543aff4b992 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b413e4be1fc0e569c7e7289f1dca8c78ef9b7972 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..118e480143d13d4ac69958c2522c6e4364dbb8c2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a310d02303ceb14d4cb741ee88617082e4bffb1f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79536ddc615fbf3d6ed30d28f5b810c8b2a73dad --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..613170da1f85dff3253bef9bf217b6f422d80e86 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd65c2330ce63aa9a9a126f41ae77225f3a69171 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf7ecb1f5aaa65e80349bd820fedf5e408348242 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9cba53f902a106782a6d8435690ff10ad8cf2fd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe767686069acbc9e8f0c51bede57ed047a3329d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94f8e311fd7b3da1b82f3e092c9779fef26871be --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54f82b3f3cdf00a80867896657a5659f82c4c862 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce7d224d5c75a43246415ddf9699867c655413ae --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..67a6d33db7f0124e6261a5f31060982f0a0e9f6d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/global_mmlu_full_ha_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ha/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_global_mmlu_full_he.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_global_mmlu_full_he.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4952391a266466761ee29a8f7db864fbfabc4e52 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_global_mmlu_full_he.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_he +task: + - global_mmlu_full_he_stem + - global_mmlu_full_he_other + - global_mmlu_full_he_social_sciences + - global_mmlu_full_he_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_global_mmlu_full_he_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_global_mmlu_full_he_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..678ee0d4c308b9e1c3cb492e088f6b7fd881af14 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_global_mmlu_full_he_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_he_humanities +task: + - global_mmlu_full_he_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_global_mmlu_full_he_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_global_mmlu_full_he_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c99b4806afbfae831bdfc9e0248aea0e5bc7ce71 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_global_mmlu_full_he_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_he_other +task: + - global_mmlu_full_he_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_global_mmlu_full_he_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_global_mmlu_full_he_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12906895890f10327ca750fc03a6f1a326e59a8a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_global_mmlu_full_he_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_he_social_sciences +task: + - global_mmlu_full_he_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_global_mmlu_full_he_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_global_mmlu_full_he_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6e76e7a87c10d74a947fdb40107148244c582e9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_global_mmlu_full_he_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_he_stem +task: + - global_mmlu_full_he_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_he_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_he_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b5efa3ef66e75d67dfb6991884c49a79662fe39 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/_he_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: he +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb197c3dee961941080d5b88f1febd1abf6a43b3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ab9ee20e2df2ad2769a4ab644f94fb05f788de4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8950b1e039cffddac8bf1182c0ccd3c9be4c517c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c114348b85622a32550d967b55c20f916ddc0eb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1324a04dd97d72f8c44b7d659b4ceea1b665e566 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cecddc603f45eb35cfe4ab80cdbd8c27cffa4681 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c0f8b5d1c1e4176d149b3e90653aa1478101ea3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4c36a41443d998c85a309c154316eaeb19c86f7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3633d53740d18674d0e8505944628df47fe1b631 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a28c592e502b2a999d8978e3173d6a3631203d2c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3893b9aa04956d4641327bf1bb53e175df65e9d8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4167874ea9a8832ea099fcf860d45acd975599cc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ee92851a474e6f93ab2fd4b9f4d6d4ea5b5ba77 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ceb32776617c04b74f1cf0897ff85618706ac6a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..00658e28e44c99519557fc8f94cd6754386aa8f1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10f2ac18dd7c0de14e1b86d28cfacce1325be8ed --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aac3f8dab339e87450c248d5ad7f5f2a7ae4eb98 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..299a73efd25f09c7a5f3d711ec5aae57c2fa3fa3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d3ba89356a66c2b305d097e1fb353fe11903338 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f67f8ef3d850e5a2f17a55a8fd4fb4ab9db25c1d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ca8b6f86de781c93eadcf313ccbf8127d8df982 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58e4081a54b25820f74518200f302130aabcbe77 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d76e3879d443b2247886eeef09bddc42d5781c7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e374511047550cf157d05e52709b408b7a8100db --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa0b7c711d20a4527886dd67260fa83b2bc760f0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f78a5c59f0ceb158b74ef7d08c22edf3d518a41 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15be9243320a614d9c5f04014ceadef1b4d1d50b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f309c0bdf72c16ee2ef039f38fd964b0f6473ab --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ae831c6048da958d651349f7e9e802c01fc78ec --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a2e817040d3c3783a88c9b4e2fe734f8560123c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c05da45aac6dba2333c46e2e7fb108d87fc8198d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b818e4fecc02f412354cb8345ecd969af36748a4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49f7ce5dad10a20012dab1ffcce2c26243227004 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91d085670e39b875eb15a8f5d9f2a3e4aea1c9c6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1bedb4f42b654c195491149b1bf8a33c169ee62c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39fe15a2bd819eef516f62dcb4582a2d0044414c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e54b58b33115f87f1f46c8d49c96c7e2753b1cf3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8190e96ace0c789304a56e2ea91660eb437aeec4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5811f804315c644c71bd0043d9f7726c78aa404 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7fe44232ba6fd70a2d2dde0ada318dfce3d38298 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c9082c5f5482d7a66fadb34c240b333748bc7d0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc419deea25f0070c98939f0fc2e274531b4a8bd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d889642bea8454eb5c7aa58fe8a7f7c9f15daa7c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1155482347a62cb9e36546c567a3f0a84edd926d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30d49701b51d7dc1942b264143d42b2dba1103c2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..458632de43041ebbfe396014a8a72e96a4d3e499 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..938356731ca520c0cd0f80735d510c4cecba4f98 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aed2863685feccdc5acf230100c96ae90492c937 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38a9e3cc2e44e6f0b68205a6251fbe7c0fe8c822 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e8ca950c549efaef2a513a48999f8b44228dc4dd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f82c289238a8b7f11f0cd70badf857cea4bab501 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3aff6615fe0726df00a3ac8913f815852421886 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e99aa0154cda8d17d602c4b2016e39166df1b4c8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de81b92c9f98341d26ebd762f5fbebf51c941265 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7be650440d82469e4a57826af165f71b2706ef73 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6f51e1b93a51544a8cfec7f37f0cde169e6a9ae --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3d10a0d52724e6f969bfd3f0f785592f3806d66 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/global_mmlu_full_he_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/he/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_global_mmlu_full_hi.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_global_mmlu_full_hi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c899be84ff3458538974232ebd9793694a85af8d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_global_mmlu_full_hi.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_hi +task: + - global_mmlu_full_hi_stem + - global_mmlu_full_hi_other + - global_mmlu_full_hi_social_sciences + - global_mmlu_full_hi_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36492fa3fc1234f1b9996ede7da4f88aa7c76b8c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_hi_humanities +task: + - global_mmlu_full_hi_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08dc16b0f679ba0948cf4acdc1e46f23adfed45d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_hi_other +task: + - global_mmlu_full_hi_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a4dfdd726d67dd16dab87e6fcda796eba0709b7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_hi_social_sciences +task: + - global_mmlu_full_hi_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a0123ae4a892a3b73962c68774353bb5e54154e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_hi_stem +task: + - global_mmlu_full_hi_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_hi_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_hi_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..90731e5b5132735e13927fdd689753d87b114c0a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/_hi_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: hi +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f239f067d718c93b757df84a058911fad7113d60 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dfcd776ed8cff6429d2646b068f585dfa8d50254 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbb6763d3e2e00df332526f7452f96bfe28892be --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5882427e711ef090cddc5ed6fa9ba10a9493a3b7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b0c6c3b0babc254cd924ccdefabbc2e62b589fc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5326c8dfb2fd48a423aecef01fe5a42c89e01de --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf9e21300a6017ceff10fe8dc03b9a2c4aa4314f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c79f4250c810f6e924f51d0bd46355ba1c6d7a52 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e8b0427b43eb4f64661b646aa8795776dfbdfe5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e8c0df2cffdc21b45426faf65d2aaa91e4fe05c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5fe337ee1fa73fcaddb77fc9979d0a0fa568ed63 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..029a02e07af38d65273bbf4d8fe8edaa53d455e4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a674897423d88080d92dd1e7479c81c96396715e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..355053b25b7236db96c18061c5bbb405bbeb79b2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04dca10d70fe4a2691e9322fa1fe77af1bda365f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca7a30832596a2a0e131a02c22ebbaeb43cb1fa6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae534fa62f271be4d8b940342f50aea81f7ce2eb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..096fd58bca90fdb6b8683c2e9b96eb62569e19e9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ef04ee573cc7d023669de996bbeee1a7e970349 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e8913e5217d8f0ef9244338d8c33ab7deb992d2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..180eef7563b937ebb38335a9c48f1750c38025cc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32abd63b6e3f1e57495347cf054c739604728d8b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1089908b082a3d6b3d93572850014ac8c16a6c68 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb22bb5106a66ec4a5214a00d348c4fa90eaa2a6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..affc27c0483782244eec39a806a1933ae2213bf7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59f97c949e8707c0e142309684c907479b108cd9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7506a4cf2fadf837cc6117dcdb557e6f251c839 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..406035bd423464a4af5061f2cfa77318654feb9c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5c2be371487841b9e03f7da21f65dee262ec0eb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a955febe331ce3cd339dba9d4ce9240561f4cfb4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a5573f8b315a79b0b5d1a9df64b1b8046adcfb1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38ce46800daee48bcb3eb6f1913c691b9295e5b4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2486301f8c7b6ab5123c77bf8fcb2073cd2d4e64 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f889885b32e68a52026110e11f1e440cb04d642 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2cb0d834e5d7f54ac1aa316a459cfa2d8e34797b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..113291306246243d6ac0fda1416829fe38656877 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e22cedbe2a477bbf7dfdb3f4eedeb78e3ea22d66 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..134ab0806eeffc0d183fe38dd8648edbdf47e5aa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e523b5d1a0f00cfcc192157c30e42ce849eeacdc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..11d8930b739dceda508860ff3aa8164f0cf5355b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad38e3e62fe7ca1b10160a3a579f243529eca7ad --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c31f888329cc7ffc34a597b43b02c009e476352c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01145f6f74bbeb7972d21252acb9ac222c79ad7d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4acbb127b8f67fb74ad04e702538141ee5d2ca5b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..889c0018207be7de113e0dba8b4d26565f91952c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a8aeb4d3cc93d4f8ca29e2b383e65cea853a61c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad80a3c086f2d069a4226834f1cb0ec4700635c3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f5477899b53f51f03731608130cf564c92b886f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_professional_accounting