diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5da1320475996ed8fa9e4777d940be24617787db --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc684975c906b9c8c9d3347672491e366bb140ee --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84887d18ae74fd049aee105968fe86a07fe6acda --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29ddf5bf3b34e4ed8ed6dce9b4d092e387d4f1d9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0680bad6b72014110f5520c4a14405a3be9e316 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07ac341bf25ee3baf8f03f05239825e193f58dca --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18e12bcd3f9aa77945f9ab99c9b09904e1237565 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c37522a22c9715dd437e61796b83d1e0ac4bd47d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbd6bf329fdb61eba991cda68125a23bc7de9035 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8766c3489d8af3625e8a7ba4c1e2f8ce3080fad3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4edbb98cfc336dab92e778c0da10a9a4580bd8d4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24cea6320aa1dd60e9bff297b89b868fe8ca41a6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3160fadc3521f556371ae8aa93a9f11a6758fa63 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8e480e68776c0581bee352a1efe3607c33768da --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a7b77a14d2ce5f56881970c134f9e6c7c7dc0fd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c71a4f2929b3a5ac670bf2df0683e81594055a0b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac34ba2035918081ac852af72b41257c4a11617d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6049ccb1ff315ef186c707530bfc999cd265b56c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d974ccfaa60f480e0fe9c9ba09427994c751d3d8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f05f7de9a9ae98daef400ebe3e72fe645a839266 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59cc8deed822b5d1c6e6b5a494adc5ce97179562 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb78b1f758f52c7597313000d322ed433043fc3b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..685bb2a4da295c53d7a1a37c09181355b655409e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35c21255fea4db12ceec3c69e560b09ff0bfb4ac --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce70d0067ab770203c1cbcdccf68a250dea4711f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cce88d1dd17011bee9b15e4a1e1bf1272157225d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39fc895382665a7a4173d70d62de936c0b88a1b8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3dfd71cc390f8120749c4a1ab99f006b05095eb6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd08ea345de134bbd88a0ac9e366a6d1c9a33a00 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef616ee16a63c4c90989ab2b394164fde267389e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8244e6562ca0668c1f24e962dc6112ab5e2febd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f4df8105ee4b6c480ee01eb0bc599e6177d59d1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06f71986d3c1113ea2ebd9d6e8bc97bc442d9506 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/global_mmlu_full_ru_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ru/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_global_mmlu_full_si.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_global_mmlu_full_si.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6503e26786fd09d671993b26a5e14317d32908e3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_global_mmlu_full_si.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_si +task: + - global_mmlu_full_si_stem + - global_mmlu_full_si_other + - global_mmlu_full_si_social_sciences + - global_mmlu_full_si_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_global_mmlu_full_si_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_global_mmlu_full_si_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b97994d0495b68b8d693e84d58e5c87d97799cbe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_global_mmlu_full_si_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_si_humanities +task: + - global_mmlu_full_si_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_global_mmlu_full_si_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_global_mmlu_full_si_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7600ca45413d90b9b4fab5fbe19118baf197e16 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_global_mmlu_full_si_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_si_other +task: + - global_mmlu_full_si_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_global_mmlu_full_si_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_global_mmlu_full_si_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e2351a28a55ce2630d7f671ca3ce2f24c083168 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_global_mmlu_full_si_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_si_social_sciences +task: + - global_mmlu_full_si_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_global_mmlu_full_si_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_global_mmlu_full_si_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8878bf808bcdc726c1f0fdc7a42997a142788c5f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_global_mmlu_full_si_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_si_stem +task: + - global_mmlu_full_si_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_si_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_si_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c775b20106a38ce2b9a42c91113dce664318e2e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/_si_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: si +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b81c580338b6b4ac0da7547f4f6848bcb3c048bc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32315245625bea5e6b01201d100c9c62847553a6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7ab9539e670ad3b93b53034758bcf295b4be2fc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8281fc42f5a688993b6ba454ce8043fb9a39874c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a7f5cf5b5e051f092dcb4c2595d177c4c24e832 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e54148da1d227907620fa8cbf7f46beeb5d3f3f2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b797ac60370a3efcaf6ca62b8a612e5a940d4771 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba69de35d301b43405008fcfeb04d180485ff27e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65ed942495ea5621c79a905d9f3a31713d5f2c97 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1418aa0dbab50f789977fb828d2b407d2bc01201 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb32cd4fda7a0bc9c2996f8c648a3f4cb23fc55f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce5ab9b7f949470dd28358809aa88c0e3048bf05 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2ab5718f2d7ca0972c6a5d46577f593df036344 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e764903b503c384c864b1c5b528598446274ea3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99679bb0ab3c3b1ae4007ba0ec37b3584a292396 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..553bc9bb7afa1e039e13dd460e20202527c3e0f0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..112814b6c669a598fbaea0218ea63a03bb3c1947 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..008b55377da5099f0c6ab46bb4b4cea774548c32 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fecd995a07e95406f4fb29070e1d385bea85163e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d3018b606e900b786d855bc8d486894802aac4c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e80a1f2c4c946b23dadd292e73cb37761b2555e8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10e15738479a5be1d7020b3d6a3be005d5eb3ad4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12d90b97f710ef2d2691931b2670792d431b101e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d285c2c6a813c62ee53f4c5df13d315b099ee1ea --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c85f2dff989cde83531f7ad470651f094e209f9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b292fa50a63c5e02b6bd9d73d04e9f6fdce4b90c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ada74f5f2d54e62798ee77e5ffa8b5dfc094da43 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84bbda2898f8832fb3e14292abf78f4717cc82ce --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c378798ca925fa44693a0c5340347913676b9c2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13758f22fb986c3e3db008f711ad5d1c2ae64af5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0fe85e14ad380875216348762842e4fa0e8c6c68 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8afaa392bcd373da763d49609ca14d963f8f5501 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2cf69a6894d4889ab4a54b63fc20149ee8846d01 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..418927d5ba316d75bdeef7cad8eaf1e5ce0313f2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de0a611da103b2f45c7175a81eafaeffd41f8b93 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10212173664e47f7297fecbbd2ca6904e0882a7c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d31372ada4d999fa9240a0ca1f8b45ec16ddd507 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e3d0e7c87c416910e86d7bd8cf59095f408bed0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4e29c9ae4d6b7f7677e049f221fde9832ed1eae --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dff414a16e6a5927ffc00d9b203515d9ea60433 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6160f02b4b9fb375cecbc13e330aba0772065f61 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de1db6c981aef6da775160b3e2734106990cae92 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d48cf75c9333e3472447bc7a205e61e7c061e397 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d08b811f33ab7b1f649e8ff37e2f1a614cd6147 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3163db497fd90e3229709c90d4ea44d7d340ef08 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f809bdddffdf4b4274703e58fe7964cf427aa63b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..964e6ab7b1d24be8c872e4bf914dc981f1c5db9e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c04e0bbc337dbe6c191bf4a8c9239b2883e67c91 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6542f14e34c0bb181ee423984cab769009b05a7d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..384489798648bbcd476d3389f266552f343d50b7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80f36885b740754b3daf6ab884473713200c62ff --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ac5169ed493c36faeeee1bc54734869491fd6a6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21423506a8ec993e4f99773332251bc3d386906f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c86ee0a346dee2d26edcc0eed25c164e5a355c32 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28c238e65078413dfcfdd1fe629b177772414c7e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a193546072886aab9584e583a2a8caa6514b3952 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..424c23c284c7df240f9e50ce6b95573c627b87a5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/global_mmlu_full_si_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/si/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_global_mmlu_full_sn.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_global_mmlu_full_sn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd084b3284cdac35d5daf20e898e890d1db229c5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_global_mmlu_full_sn.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_sn +task: + - global_mmlu_full_sn_stem + - global_mmlu_full_sn_other + - global_mmlu_full_sn_social_sciences + - global_mmlu_full_sn_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69690862098b99934a3a6e85bd365b783c230219 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sn_humanities +task: + - global_mmlu_full_sn_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18e750b6291313947dfd819a88bf26893ed31801 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sn_other +task: + - global_mmlu_full_sn_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8e762154e8c4e6b978d306c29124d6f682cf4bd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sn_social_sciences +task: + - global_mmlu_full_sn_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3136233b403abde2ee87f794edc49c97edae5ba --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sn_stem +task: + - global_mmlu_full_sn_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_sn_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_sn_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..30d50ba0c935327da78026e4056a8fb9cb0389ea --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/_sn_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: sn +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4de495e9abfd59ec52b42a3697931f079937327 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ef227aa5f15f72002312f64a8239ae85637981b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8662ab9645bd99bf373f765af1ae1c00f4f09b96 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f4741c04674ae249c9692210288c7f1182a39c2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7477170ef5546c00cfbcb87c77e9bbaeb97d9db9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d0ec277caa41d916bda6ea04f25722c08e05ff0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f0c4f42643cc5538cd187f098d1108a6f143653 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8651ee192a5024abe1ded9effe39bc4a8dcb8e1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1d1a98e0ff81247eacb8e1a82d035ac3fa6f1a3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9ce08f307e45ee2da6b4ef44a43ae03ce9a89e3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae34a82aa856d75c5ec740c2575a39ccf10e5877 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b41c175ec6555a7277892d9acd598dba0e8e817 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5aaa8a782d361a1a777ac2f1755e2fd7360f84c0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8606e96c2959fa67458306f1301efc25cd7e54a8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c57f7038f6319d5af8bf84817d2f26421db5b41 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ed5b400c42a3ce553cdb51631041da0e67152ef --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55dafc2bf4a6b0bd2d70b864f0eedd141c2cd8c3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b8ee96f09618fa6f2fbd3ba4aea6ecffaacacc3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2597a7d75e9149fdba41fa09d8654db5dee697a6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e6be4e6115788e02ce217d961420d6feca650ad --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..446da912c7dd1e8bcf96f3d0c0b21647050191b9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd8cf61c4f32a26c8a8ec9c11c24a64a9442c3fc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e178adfd6e82f6018026a2f282c42af949b8635 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ac4efda6b728e8b874ec8415a0975526ab7279a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23ca0b41766564f80dfacb764131c8d962a7d400 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0bd9be1952012c77911a642bed92692728807198 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..916e14caeb648a2fc2257e1508426576f1859fa9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6a3e60c2c54c2f6d92fc2713f4053259862df94 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62a197c0eba0a301c435c9ed68e1f108c3716022 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..815cb60b16d7848b05b40c6ca08b871ec483fc26 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff9f970e08d1206140a9e7726114c3a001edfa99 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2dedc38d7e72ca888b3551b0ac305fe8b3cd2a8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ef139307183fe54eef709b0f9740a0f9e1f5747 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a52c2ded23646b6c837a6ef9df49288e09eaa81b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..648c3dea0f54f303d76bdb2ff66216675813611d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca63c411a7199755284772a8fc6e451e168b2291 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d74a7f18eca74b91c10147d9bb3dff3f775dd9db --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db272b3b4784e40004eddd0aa250559b7730353f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db3bee4d44a962c6f5eac646b936d8d47c2d0947 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a700c4e7c59ce9f47749d0ba5b2c6fc8b66d44ef --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b826b187e34201af1e719e556062f2cd1f99a466 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dea895aa9bb6a543368b589a320c44d018b4f59a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b641f6b2afbb637d3fc41280003b62c196df6405 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2951a953c7a42b7c95e7682abb9ded91456a0f69 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9816d8b480449f7be8af8c1d57ad45715e9ad447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ea1050502fe65602c7449e16c395c33fabd927e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e941437beb64ddb49832cd3883f6feb003acbb9b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..057a197dbf9033fdc09890a2848cb5414c259c0a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72c9fac75d480eb4c0428f9b505f12918088e02f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e727b3cd510142625bd0a44c3c399bcfb0dc5f9b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..341322d2c83c20120ed86f0dfb0ca0c28940dba5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5448baa47394fd79fd7aa472cfc1c155069da575 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..542c709af46185561d0785a4f58f0ecb484004a5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2913db53bde1f7289e7b40ddb08c01790ad8a84 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad4768478d16b0fccf2e88dfce772908b7cf993c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..254fedb4f7855067d1d5c95446821f4ed89e20ad --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2aef6dfddb867d78365e083fa0b53835b146aaf8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/global_mmlu_full_sn_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sn/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_global_mmlu_full_so.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_global_mmlu_full_so.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91e11b88ebb1db0af2687b1dc2057020855b6286 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_global_mmlu_full_so.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_so +task: + - global_mmlu_full_so_stem + - global_mmlu_full_so_other + - global_mmlu_full_so_social_sciences + - global_mmlu_full_so_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_global_mmlu_full_so_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_global_mmlu_full_so_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff78bfaba252a3db2b0c3c028440fedf971e92b9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_global_mmlu_full_so_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_so_humanities +task: + - global_mmlu_full_so_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_global_mmlu_full_so_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_global_mmlu_full_so_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eec8e66138735a6abcec79f10a5888282d25bd40 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_global_mmlu_full_so_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_so_other +task: + - global_mmlu_full_so_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_global_mmlu_full_so_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_global_mmlu_full_so_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d00ea1f54b691cb797244e9941dbeab2d42fd9e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_global_mmlu_full_so_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_so_social_sciences +task: + - global_mmlu_full_so_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_global_mmlu_full_so_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_global_mmlu_full_so_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..497b9b01e352d37ad1a5fd52757130d19933810e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_global_mmlu_full_so_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_so_stem +task: + - global_mmlu_full_so_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_so_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_so_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb052a633605159b0271f62d9ed512d447daa0c6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/_so_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: so +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afb5d9084c83935d297c6e120634c9fbce9705b7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79f3446df384b530c631438e174bd608e077a610 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54a2faa0913b44a6896c461538f6e92c67727393 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65bc598cb2a4a5af6c008f1dc9684e8e66f5d448 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..224aa39b24795ce983e3319b3bcf3c708fa50d96 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..758d22c37d04b323028098313c84dad1a9d6f716 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35c224308b993c3330d8dc5963264be1ae2ad73c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86428ae8238ec533e73bd0fb73cdd48c527a744d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9957a2349fba24fb2c0fc9eec0d4fedf687dbf9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f51a1b123d8a47d5246cd8d3f8d72f6ddc47320a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43388d6c8d35e62defc815b972bd4ad17e2ca6c8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a5563308f90a715c3216218691a957cad65fd7a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97dfa147b60925a7f249f774cc40c7185bd7be28 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9792659fac1ee2ff448715a5491de7204efecb93 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ed44e41f83996cb93084fbc4d3ba68cb73e34f2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76628481f9bc0a4ddbc18018f0d57c9671c71140 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b7645c8e4e52090be2305b8c726dd197e1408e0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa75e6664fc2609971850c75df2c5f0f8a0534e0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3ad29d4c23a2e0940832f9876a7aa8bd014b801 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..274af23bfdcec5f12fd72bab441e36ba30917325 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6bce30d68a11677821ff5996f696d263e0f5eae6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfc44f08d6cfc94eb23c6c771cd5634684fae266 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55479c399128600b0f78b28088ab4846c25478db --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ceb5a7014e29946b3ef4ceee91a092c78d44d556 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c403ec513ec5f52a27910ff80fc079005a8da6c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e8089bdf87c8b494a4657c15400ab1787ab1cd59 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32cacffea668b3cd783bbc3f05d8445009dd9c3d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd2c35ac8eeae6ba49aa768df65c516914b04e78 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26f2cb3c43fc9d84809dd05ce071bab3280abe8c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..730075b1c1f1afc011e4ed7997cf0d4111ad51c1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9702a661bdeb4535b3f793b00031f957764c688 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78a21d5d972b73e3eee5b33fbc0482506c7e04a2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c95b5562a636c817a5b9e84fce5c1116fad9875c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..632778d3b276ba032f36fd553ff1d0f86c211661 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d5ab1c53452229d9af58e781357d2f206baf4cf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1372a1d17cff381788856f6cf5382305a727ad0d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19a1120eb85f9e72ec856ed88a3eb9389de21088 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1e13ddade10f3e6b90630eefffe85e3f5c8a506 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e325205640a7355d66d25b7aeb3f0bbe7d5eabe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b1c002fe5a8a6a05ea9970e752fa5b9e21f1e64 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0136dc694187c5bf3732c150eea327f14a80fe0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b8a33ba92fbd59ffd6ae31ab87e82c42952ccb9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1bd0011b8bc226e01e5bbd52eae40d09583f131 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60418a65f9377ac7aaa3d9056c75e996dd5c783a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5aa40241a4b85d9d7420843cf34464a95d21b5c5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..421a98016b83cdd1794302bb2e25f1daa5e25787 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..721bfbf22d876a9e6f17d41bb05e55782d442995 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ca0c5c9bcb3216c7b301a53ece3c53e4e47e1fc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f57b5949641358183bd1f21cd5dd0277ecb36d7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7d6408e291220d4e221132cb16dbf948253520c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a03de5bb99427c9b189597be049277b5614de156 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7af81e6ae2b4e2d0b78d31706cfe663b0690418 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b52ee25901816e8015d4a83d458295f675582383 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f3847e67646ebe82f894e56ff7ff882bf764c13 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6017167aafd4e39a0a247494fd2c578d9da8b46 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dc85b324be8c45c7b59a01d1cd537a207656651 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ca99e5b4eeefbea4f282e76411f7ef7c260d038 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/global_mmlu_full_so_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/so/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_global_mmlu_full_sr.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_global_mmlu_full_sr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e0c55a5d231c1f4cff0329843d91f1679095f53 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_global_mmlu_full_sr.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_sr +task: + - global_mmlu_full_sr_stem + - global_mmlu_full_sr_other + - global_mmlu_full_sr_social_sciences + - global_mmlu_full_sr_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..080bc545cb3f9e0d6ac97f8d7004fc763c507ae8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sr_humanities +task: + - global_mmlu_full_sr_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f0735eb4439896f0cc29aabcd23c57c0a90aa4f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sr_other +task: + - global_mmlu_full_sr_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bdc29d1f608c6ace5e8c7475dd1c6c27547d5cc6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sr_social_sciences +task: + - global_mmlu_full_sr_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c4aa636944898c99dd4fcf7e42035a978932eef --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sr_stem +task: + - global_mmlu_full_sr_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_sr_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_sr_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..6af61b3b28d14235a72fa8b548da96c7f73c5a49 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/_sr_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: sr +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3275870e203bca8e21570f6cef1996b412c0e9c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5689af7369b640521929adde0bbbdac8e64478ed --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d23a4383f0e30c1b700d077fe7753f4db44310c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e89f5e61be0e2f111aa5a2a490fbf18a2e570756 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5611c15c469aa2e4b0471bfa4b79893f9f88f21 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e28c3034aad065087ebb8993bbffab306f9b532 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1eac952c9e44e97a7678c648c5b917502c9ba185 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1146aa117a6b8049b105f8b65a2fed6fc374fd8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bcfda2ba34dd2c8b7c7b391eec67ac3a6bcc7cfd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3beb5b26c1d4fa89b5e5b91e7f451baf016976b6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f959a02fcafa5915d1e9c826c956662f2be8b393 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e8761e0d3836bcda9a1f1f4f0fd5ca828ff775d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9325f6de74b826969db631ca40120e821f7cf1a3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc4a5bcc3cc759d0df3339a9a558ccf31cf12553 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3a5a78b53d9cd2918383a6e8385ce8c6dc30504 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50f60166c482fd5721e5ef8002f5be8d2b40f5d0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8bdd854f1ab22207209541dadb87085cbcf9f0c0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88862d212b1969fc4f02a0bc875b705db9f6e14f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f2b2952b6706e4e29f6367da93081db27338cf2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b89deb12e19d7581fc914e84be334b90a666985 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55fd7e8e662ef66d2892dd16ab1372ae9f47028d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..946acf0ee2a3b5defe39e075158bcb7326ccfb7d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0705897153d5d0b16df90781ca0eb75a2ed2cc78 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9721c9ba99fbcf90e560beba130c4c7ed55e234 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fedea95a6bac0ee70fb0f2f76eddf37af23c8ef1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dca9e1402fe5a89d4363f955db72b8a2efcc1ed7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b01276f650319b6ea97b1d63b9c6d46e931e0a64 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f549f8ac2d25f066495e773c2c28b46621cc7fd6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6b31eeeda41b4d6bff48d23940b452887f45d59 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12d0f0e5c76c2e0b8c013e2e2d648a845dd3dbfe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98c401001b4dc83725084429c694cd5dddddaf8e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76e6b45c8624d95c6e9eb80b17d4ddeff2ac2bc9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0ff1d95afdecd3c47ab8f84bd771631141d5874 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73a30099c741f49d3d5fa3ea1897e9e9e4999aae --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0aea0826f6db54bce4a14354baa668c1097c7853 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..debe604fed0588bcdff83d7d1b40b4763cf01dac --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..407417f3ea3e2583e01150b5d730c8128cf990c3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..513a7f8775ab9ef422e32782169ff829d32c9950 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fca9de044cfa215d365d6edf7a18c0e5d41dc192 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8267563e63768d3d500c19a6741057e238cd0ef9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ba860f2daba6111a78a08d811766891c985a913 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ecdbcea91d15ecb8612d53870fe16ae82c486834 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54bf349162b1417c8f0a09981a719fd09c23bbf9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2eab8d4a06dad83bcb05acc978f8ea6367f122ac --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83e1b84c46f4aee1eee31061b08b58aa8770a1b7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..654ee86b925b233e1ef284aa834f6a1968cb29fa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a2f944b7bba7f8d342a3a1bd20601fd2d5034e8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..648ae0cb03e7f6a77aa29767a784a748b3d7b738 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ee8a831bcf754d7d0f5dd005c5dec2f08761017 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b14211541c392ed442de13c5ff78c467f44dcfe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19e2dc544db339c6ce6224aa2062f4f5a4464930 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..043024c0dcd90635962ee2b4f03d360c80440932 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24720925a41855ebe53c1d25abb4f510d322722b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc93c5e07a46d3a54c9575ac7d51e618b8165e9c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b338dd6b4034b6f47ac59e2dab09215f28161dd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b07588ad15476540ecbbd018e013fe3e7de1e969 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f78403edfd7b801675daf8ee10225adc82a114c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/global_mmlu_full_sr_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sr/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_global_mmlu_full_sv.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_global_mmlu_full_sv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..078de8b63a3274de6e00b625f3610f8167ad63dc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_global_mmlu_full_sv.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_sv +task: + - global_mmlu_full_sv_stem + - global_mmlu_full_sv_other + - global_mmlu_full_sv_social_sciences + - global_mmlu_full_sv_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8b4628f754f5a8ba65b147882160704072bb562 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sv_humanities +task: + - global_mmlu_full_sv_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b29ca130b28e39600d70f62ecf40cdc479ca6ac --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sv_other +task: + - global_mmlu_full_sv_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c4a813e8ceaf1782775bee7eff3e770ce07cad0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sv_social_sciences +task: + - global_mmlu_full_sv_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6fd88f1efeea969d2dbd28d1aa842a92a7b413e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sv_stem +task: + - global_mmlu_full_sv_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_sv_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_sv_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b9fdea988da74442edba95ebc9f78226588f203 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/_sv_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: sv +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8329302f55bfc6f86e6db0360698baf191cb3235 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac9fa560ea78ab74f23107bef64503d6b5d26c05 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..096e0e8f04e8bbe558b173fcb955c1944facf124 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ced0b051fb4b37d78daa7756b6b73e708ddb572a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_sv_other_tasks +task: global_mmlu_full_sv_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a88871b4efae57ab7ca198ee0d23510a4cae5c07 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_sv_other_tasks +task: global_mmlu_full_sv_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2462c179f1bd149b94a45205266924741e65197 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ae3fecd44f725570ec275f5cff43bfbd8494bc7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3f00b24915c4a2bcc84513e01f4b6dc8d997e76 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..71f613d4a7d5fe72ba2e740a8f7e4111971b076e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46f4c6eaa2efdcc411b57317e6e4aa72c8003966 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_sv_other_tasks +task: global_mmlu_full_sv_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06906bfd32e6e785c173305d13c272fa07c4fbce --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1013ef3008caf86e62f11ef8bddc73b8bb36386c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6a752f0ae863adf73b2f0dc2b4135af73e17d3a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..547365f63bec67b0383ea7877bb89e8897118d42 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_sv_social_sciences_tasks +task: global_mmlu_full_sv_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74086a15a5be1c07a06732ac8a835980b9b3e0d5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d1f4847c2f7551b9bea3422bc33dd68e67af02a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b78b5846b90af87110e424fb6b58c37b7cf810f6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_sv_humanities_tasks +task: global_mmlu_full_sv_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd2056294b1ecab4b0c9f192402a98f66adb855f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_sv_other_tasks +task: global_mmlu_full_sv_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc6ebf2fd8e0e84a1558b492d51a5bf17cc1162d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03773a83b86832425da35e3e0caad4d6ce290091 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3db653a9e175335012b189e8c88c425b1558409 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a087557a8375f0a4019faa80137b1d4b77884d3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_sv_humanities_tasks +task: global_mmlu_full_sv_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..638553848b693ef8b8748686587f8cee537b9a43 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_sv_social_sciences_tasks +task: global_mmlu_full_sv_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e62f26f0e8018c11d9e14568999441eeb7add15 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_sv_social_sciences_tasks +task: global_mmlu_full_sv_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b686a26e66388951b274be574fa0e19970f82e6a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_sv_social_sciences_tasks +task: global_mmlu_full_sv_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..177165383cf0042cec3134a21eb4fd634988164f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9817c173e56964e24e44a9219934a0f124bc147 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_sv_social_sciences_tasks +task: global_mmlu_full_sv_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61359149c5df8d3a56cd6efb600de7cf2a7b1b51 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce3aa9e208d35250594547e3715e6efbd8bbe088 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_sv_social_sciences_tasks +task: global_mmlu_full_sv_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f705f8e19439cd36ff0000463465b300d24442c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..765cdf6033fb2690f0116c3c62a60a27408c3350 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_sv_humanities_tasks +task: global_mmlu_full_sv_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de7b30b4ea8cce8a4db811d3b09f865df448b5e0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_sv_humanities_tasks +task: global_mmlu_full_sv_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20969051b83299571780a0ad14da9f25e0fd5c15 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_sv_other_tasks +task: global_mmlu_full_sv_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8bd5fab3cb03985b49a370120ca30145a6a33b0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_sv_social_sciences_tasks +task: global_mmlu_full_sv_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e5ddb57ea3a13fb8458a0c022fdd13aa7fd3194 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_sv_humanities_tasks +task: global_mmlu_full_sv_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff161d5f8966e56878c109b43e0071d973ca9d8f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_sv_humanities_tasks +task: global_mmlu_full_sv_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1602c9080495901a48463178f4dcdc07fa17cbe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_sv_humanities_tasks +task: global_mmlu_full_sv_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f011063fa668db2436819569546b544d6a42765 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ff7b8736be311252d1518f6afe1b16e19b4e379 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_sv_other_tasks +task: global_mmlu_full_sv_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0e669f715bdadce6a32e4e1fc9caadbda44296f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_sv_other_tasks +task: global_mmlu_full_sv_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83e524457a12ea576ea2b27d346da963f4792c2b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_sv_other_tasks +task: global_mmlu_full_sv_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f179879204c6ae29679e8c621a3a53645d634442 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_sv_other_tasks +task: global_mmlu_full_sv_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f03ac0905f2d468d5dda67b1e40f3abb77652ab --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_sv_humanities_tasks +task: global_mmlu_full_sv_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe7f58d23f7b10ff6e37ce5252bd27e887d38b98 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_sv_humanities_tasks +task: global_mmlu_full_sv_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79207a8778257e76da9807c17342ee22ae8f89a4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_sv_other_tasks +task: global_mmlu_full_sv_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae533079dbfb8cff7990659c578c3d8727cd6ff5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_sv_humanities_tasks +task: global_mmlu_full_sv_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c602c4fe434411f7c9d54d3320de84866812406 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_sv_humanities_tasks +task: global_mmlu_full_sv_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebdef8a89134d3c6f7b59d052ae5ada15e822a45 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_sv_other_tasks +task: global_mmlu_full_sv_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3645c38a3c17733b3973402b895e7dcec371af84 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_sv_humanities_tasks +task: global_mmlu_full_sv_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d40f577df995fbf77861b82593640a72505926e3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_sv_other_tasks +task: global_mmlu_full_sv_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..edf831060cff4867616f85c84db86138879d4224 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_sv_social_sciences_tasks +task: global_mmlu_full_sv_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f897662c221c58626723a6a210fdd8d473d15efa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_sv_social_sciences_tasks +task: global_mmlu_full_sv_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ad4fb5ccca32ba3006460ea7bda2cebe8281051 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_sv_social_sciences_tasks +task: global_mmlu_full_sv_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b869606048121292efb01118ccde1cb13387a06 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_sv_social_sciences_tasks +task: global_mmlu_full_sv_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..522778def3e070d8b2e1a05ecea4cedf77cca154 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_sv_social_sciences_tasks +task: global_mmlu_full_sv_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b3cbc8ddd76608a902cd6602fe94504fc1d5c18 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_sv_other_tasks +task: global_mmlu_full_sv_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d7df52b09b2fb43f64c54fa1c45949f8dd0ac5f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/global_mmlu_full_sv_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_sv_humanities_tasks +task: global_mmlu_full_sv_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sv/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_global_mmlu_full_sw.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_global_mmlu_full_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd87d0682a8c35bac720fd6b5880c72cfbb921a9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_global_mmlu_full_sw.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_sw +task: + - global_mmlu_full_sw_stem + - global_mmlu_full_sw_other + - global_mmlu_full_sw_social_sciences + - global_mmlu_full_sw_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02168dff1bc799a208fef8be525788b0b2f7b10d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sw_humanities +task: + - global_mmlu_full_sw_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9fa28a162527a99f56785aba4cea362fedc5bf48 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sw_other +task: + - global_mmlu_full_sw_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad31844234018225e5debcb135d42a76b5ac15f9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sw_social_sciences +task: + - global_mmlu_full_sw_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f23cae88e4d93382b83171bcbebe6590fb79657 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_global_mmlu_full_sw_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sw_stem +task: + - global_mmlu_full_sw_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_sw_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_sw_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..58cf53220fea417fb87ebf1d4b43a8b4c1242922 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/_sw_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: sw +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..187229fb56e6ae19611cea4c7079c790627566bf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d0d4c5c59ed00554e2ffa4590552692986b7be7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0639b390525d4ced14337cf79660949b7662e235 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a729c9da06b3121f2b8e90d56e8499b19cb0636d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6b83623fbbde49a899ff3bedc1862b5e95f302b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1856b934331b60a4e5b3a0be88aff52db31dfba3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ad547ff8d1deaf7a83e328656caa6d4c3e739f4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff8d874199ba390086adf1a621a39095837e8ef6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02f53a4a1d1d8fc8dcf42f3c0da9e4a04e6fc253 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9f4cc6cb3ed871171d24660627f5211623818c4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bcca5b3f91f2bfd38966bf697698c6d774eb9242 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..434d2faa5df8bc81027255a292dd54afcb698a93 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c1c9d41e9b2495490e671f3bec8de893cac8353 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a907de64a9bdddbd2314e6a6982ed76c4971c1d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ae86a7c22d3a492558316da54ff85c3c7e30e6b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05871f2506d86094895549efe9ddc4c2c0d01521 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d0de4076560cb09521d67e3a7046e8b172b6bc3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29bec055e9ee26367934bfd1b9c6810823cf2f2f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e49866a9fd8095d432a0ecf8e76076b1a3d100e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7adbd97813163ad8cc9c16683c5d74eb33b388a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e65ab5ab146c34bcb320d6856b08b30f7068549 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7352ad72e045f64481b102b5a66f3ce1b103992c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..797932baed3c4a8fe590166205d8cdbd839146a3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..602d71ffa9f91315b41dfa7692e012a36d32fd8a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a91dd829b9d2b95c06b3e2f71a5b567293904adb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c19b28da67c861bca14cb1c17212859edf3449dc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a9c63bd08dee2dcc355e61521571994b439daa0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..239eac65f512d431484ce0f4855b1d591f434f86 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4f19d84b37e3ac4ee4c2acd5a12ef7bc96f32d0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5725af6378bbc42534a427e152f8181b44e3d5a5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d0803407b33d3e4eb301dcd6f05bb1ed2a5f753 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfe5a9e7deacb57b19b2f24b8c1eaa4d7267632a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba20e932f12656fce0202f37ade43ffe64e7a6b8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4609bea09b55a4bc77a342165b92796454654470 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbf616b1de69f0ff77b51f233b6b85c5ce7d7ff0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6781f2d521f0dc319f7787e98af1298f574b8bf1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f862917fb66741ab80323344d887a8ba48b69d0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9eb51cfbcf588a249a71d3ad23287b30730c2391 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b0e9e67337c2fca6691c66a6731d34410eadbf4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb65e87ebc6d552982d19b70e2635771e2ec55bf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10d4db0d87c9bab0d6d0ce6760ae4e2b24010753 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b337d0ab8f133bea427f4085e0d413e01e6eabef --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f44bfa0d9db24fb91020b0989eaa9ffd679ee4e0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eabd5a916f1858cd7c2c6c76fe5acd7c0cc34504 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41c6445800ea3d53696ffbe3b3434b423f836618 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96edac9941a503dd29255f3d57047bc630b4a142 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db94a2ff4287a908026e9bccf17273e5b2b8ecd5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7cd19d35dc189953db225d5ace390fa65f9cba9f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9434ae4c22f8213987379f93f014a1f36a41d173 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf35b9c644fa3a50292cfd5c6a998a68b277e5af --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7570e28899b60f7dd0c8a29542f28cdc802e986f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54c094db40c4f586ec3f11cba73f466954a47e46 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8d5a42c00d3bc9084dac75ab79a932c7159e521 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79d51a58d088f82ccb5c7b58dce7e73654ddf3a9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..523b1572bffef3758a6910a19c713687321255a7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43179ff8432de7e09c640a2f9e2c238f7fd74968 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bef7b7f84ff14e8618af1580bf5c36f06198fcbb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/global_mmlu_full_sw_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/sw/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_global_mmlu_full_te.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_global_mmlu_full_te.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0bc967ded733ecf313badd12461bb249fbe6cbc6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_global_mmlu_full_te.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_te +task: + - global_mmlu_full_te_stem + - global_mmlu_full_te_other + - global_mmlu_full_te_social_sciences + - global_mmlu_full_te_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_global_mmlu_full_te_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_global_mmlu_full_te_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a3c479e8cd65d5f82c1c83348b3734c75fdc5c2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_global_mmlu_full_te_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_te_humanities +task: + - global_mmlu_full_te_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_global_mmlu_full_te_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_global_mmlu_full_te_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2932844acb80e0985e189581313d0b0e38a90688 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_global_mmlu_full_te_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_te_other +task: + - global_mmlu_full_te_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_global_mmlu_full_te_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_global_mmlu_full_te_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25e721db7e6c9a47d1863266bf44755e7d4edac3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_global_mmlu_full_te_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_te_social_sciences +task: + - global_mmlu_full_te_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_global_mmlu_full_te_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_global_mmlu_full_te_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe2426cac5db970f59b9ad681009a105b70de435 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_global_mmlu_full_te_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_te_stem +task: + - global_mmlu_full_te_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_te_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_te_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7b1190d48984e97737ebdbc8e1746107f6f2da1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/_te_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: te +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e922fd08d1826ceac13c9a4eb536a0f5f26443e6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..00582018a182c2aba17feb4bdf62b598c6b631db --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bc5e76ef4abf6f8296da1c3612505a9e036f83e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b440102c86fc00b4f8d8e3a5286a3f8392e172f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90e5618441a6f0284c7d1b86186c076528c717f5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f036e60d50f8e4c2d98b032ac3082cd5e7ec9ef --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ccdb849a875c73c2030beb2cbd0d70f0586c76ec --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f11e5657ba687bca3548e0f1b0e4dbae7e79f2eb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5022ce2981b9612ce5583b4d23fbd1a57a0a6e7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd5219f073e7105a9177f562378d5bbca12070c3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88dad05a2d9eacbef66427bc6e206a385da99204 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e8f37fc1dee6cdf5a64d3f52408342d8b15d4fc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f05276252961898828e55ceaef16b767b1052a9f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf008a6748dfe708ba6bcf35e700a81579751db5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97169e931a07143e4d1bf31abc353f5d24db506c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3edc89662958185b6737382b6c76fc0f814c7b2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4c182d1d071d10898010a47da30b10f75ad40eb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53b52f4d3ff79686c1018b6d8706685710316d02 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f02170ff8d048cb3502e01966e6e77c3ae41f25 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c77d30aa2dda080c2314cda0ffe5e081276e7faf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f388a0606812de534f7496e7803954cecdd5036 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75d54d72bac17a0522dabbf2a7aff9830e275e16 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..383596ff024a193f5a94346128eb9cb0e6144a53 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8db56a85168086d0de6160f13880b6f88eb96e2f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd471b8d2e4cf4b3d992a03680c51f20bd817f0d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58f577ed1206932c53832a5b8a3f4bc8f139a126 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..400a3805f815dcb48835dfb5b2c8fce407c26350 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..694ddc304f0632d3b5b0403ae370694503ef440b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b900af19ccfe1aa095845f5a5c52db1678c9f9fe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3492e7249129132ce1f659e3631c43eec49fc7e2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48a2d75a6714e41b9f0466edb89caa4c409e3d94 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e95f7ea19dd033356bac2d6385e3903c3fc71db --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc44c1b0a1f1c616248f3b41da15ff2a645fddfb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7631419b702a0492a4c93b912b213ff3e879cc3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c2c7862fbb8689e3ae5cdf3675b25b70ca395d6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..718cd9fad0c2a1b5d966e4a0b4650ada3c3ddf71 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7bb9170ce69de22eece49f97c7cf65d2a6394d51 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..123555383b3e423607fb92a85dddb5fe66a28547 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f092416ffbd3bf7b46a8e31ded9971e2c88de79e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15b84b4678fc5868c5e41f18ca00af67fda8dfed --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f0730be97277f9e1d94caa47a5915dc8236a8c4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53487f558acb7e033c2ef7a2e7a71d362256a10b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fca8df9b7e1b62b0a6133511912754e4f72927cc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d87f6b029e595bb6a81d110b69e6432b985176f5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9348a76e2949054375aee37c3804225072ecf503 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8efe8d97b27ea77899c165997d641229742ef5a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b702542ea997902dc6fbb8b83257e3fd034d3da7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..045b6e1ca5ac2fe20c04eaa51d335cee15f27a1c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e5fa30806275cc5c615d58537fe7470e16adf45 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4ede33f00c22cf9cf0363f15d2b6f742e5ef5eb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb1906d415899f46a26ecb76e36faae1d9e82615 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ac09ce0d6fcfe9ec0d38bcf616aa97532410da3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbb7bc7c81a4be3d388675c964b9f21319ed1427 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e080e0823b88d4d6ce06e007dddc376f2d50a621 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..338f080950aa736aac2d7b1ff63d990dd4a467fa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f5e38a9eb80b76a0e48c6528aae8844dac25a04 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4da26e3e17e0370a9f15543ebcbc31bca0384e7d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/global_mmlu_full_te_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/te/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_global_mmlu_full_tr.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_global_mmlu_full_tr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa4ae63fa9d7a1243a4c6bcf11f4384f99912087 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_global_mmlu_full_tr.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_tr +task: + - global_mmlu_full_tr_stem + - global_mmlu_full_tr_other + - global_mmlu_full_tr_social_sciences + - global_mmlu_full_tr_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4dade158eb2ba0b672ed66d09b205db8cb97a67 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_tr_humanities +task: + - global_mmlu_full_tr_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e80a5b9d8e30e0482db4d433181e2d3153c3b89f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_tr_other +task: + - global_mmlu_full_tr_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56fc20e14c019a09a3da4196c37fc0d4c6ab69d8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_tr_social_sciences +task: + - global_mmlu_full_tr_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51f9bb3d90e89ec77e86d89d22e9468687e0b08f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_tr_stem +task: + - global_mmlu_full_tr_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_tr_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_tr_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..e322bee67158028cfafaf5f066b2b06c5bf189d7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/_tr_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: tr +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e8215730fe3086fd17da2c6d3705f3df94c18cd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..444402254c31c5d20abca3a14610143a493a543f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e85390bf1b17894295e767c8f780190cb3cdc286 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b1afc9cf248c4896286603c7eaf40e7b5edf23c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bdfa69e6dc4ac181408c1803a777c619f551de00 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df43a67ce3c40d244893a9f723665da467bfc762 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af2b8b3e808a26211d054ffc36bb66bd61d54011 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..622854f4d1723fe8e9acea54cb61a2ae505f055a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..902bd9c1b4f8629f687057c0a3001912c3d375d9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b44d0d131520d179f9897eb08eeadce7483d6bc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27540d97f0f5e8629f49353977a2b37fea566096 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbcabeed0e45164a58dea395c6c662da5f6b2b5a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..628a4fcf089c8b34623b684ea201d0e28bd9571f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6feb236f98053fe44b87c1d45976d58fda64c777 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_tr_social_sciences_tasks +task: global_mmlu_full_tr_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a2a8665cf779c170519fb6a1c7aabb38f6444ad --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ffc6dee79e3f7acde296c2a68ee3420b915f9a7b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77c189a0b016cdf5c7bb63469f71dd108fb6e622 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_tr_humanities_tasks +task: global_mmlu_full_tr_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a756d102e745702cc33e49169713a1bb242ed452 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51e7dd9e92f19f1c849b839f30fa78522254ed14 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..077476ae73e836b89cb02508f488ee6b5479fb48 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb60e0425f02bbd1ef8dfb26efdddff3c2b489e2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b989e05875ad7f8d6f00c3310a1b6a7185fc85d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_tr_humanities_tasks +task: global_mmlu_full_tr_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a0c4d90049dbdd211e3bc6e261e573b67ca54a3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_tr_social_sciences_tasks +task: global_mmlu_full_tr_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a585f02d02ba4fca070b527b5b657ddf2c8a8f7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_tr_social_sciences_tasks +task: global_mmlu_full_tr_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f88e98310ef3623c581a6586d4d2c4d6ab83a765 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_tr_social_sciences_tasks +task: global_mmlu_full_tr_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e880b0b53d293fe4c081a59d58c5aa9a05fcee14 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5527bed2cbbb08445df6ad5981b1fb1651d60f0e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_tr_social_sciences_tasks +task: global_mmlu_full_tr_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da93a96e1dc65b23b7c784c95a84c54e3ac68575 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a28e110c3292adc977f8e13941ec391b718f81e8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_tr_social_sciences_tasks +task: global_mmlu_full_tr_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93871dcf8d6be01702b51095d327480d1138368b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..507a4d5ce35588f9e2ca5288c178ae5c723f0cef --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_tr_humanities_tasks +task: global_mmlu_full_tr_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60cc713ef73b9301dde1c5916a85e5c1906d1236 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_tr_humanities_tasks +task: global_mmlu_full_tr_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e48bf12280d7a77c5426616a80a46e4f20f448e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84a958509c079c10c0c7e269d3cf971ba8902987 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_tr_social_sciences_tasks +task: global_mmlu_full_tr_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0dc429f01794d584b3d7ebdba96a6e9551cb724 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_tr_humanities_tasks +task: global_mmlu_full_tr_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ea3b7a5158ead9ded2196f661459537db7db7a79 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_tr_humanities_tasks +task: global_mmlu_full_tr_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd61d7d741942887ae8ddce5cdae383d8043c3c9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_tr_humanities_tasks +task: global_mmlu_full_tr_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0e785c33123646da2c94dd62444305d7b625850 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ce0d753b00251e19fdff128273dab558bec7f7d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ffd4986f72dfef6928f01151d6bccbdf359580a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43814b40c614703757b39bf6992f9e53b27a12d0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e21cfcf6170cc7a7d2e1f8d1ab4d77ac6fa9634e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88fbfbe2ccb5d2d5526c15f46ded4366f7a5d8fc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_tr_humanities_tasks +task: global_mmlu_full_tr_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f92f855dc7ef6e29bbe11749c622b65db1d35b7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_tr_humanities_tasks +task: global_mmlu_full_tr_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31b39c38608bd004ef7bd2964035ec854cc0a257 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..283a2b8968965630dc401c9d912806f0086f6b58 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_tr_humanities_tasks +task: global_mmlu_full_tr_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4c170145f91260b2271a0ebbe9cc0843f571612 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_tr_humanities_tasks +task: global_mmlu_full_tr_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c69f14f76e8e39a690734f265f05a54d18d33e8b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f5e97c69d33697fc1f6eb2c9036a43173abbeb2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_tr_humanities_tasks +task: global_mmlu_full_tr_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..00a5f32aa33a74526256172b0999037c86ece8cd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8571bdb0583b29aa696debafafe4d2ba473da37 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_tr_social_sciences_tasks +task: global_mmlu_full_tr_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..539f8da66b7f6ad9bdb4e75ba746477b492de91e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_tr_social_sciences_tasks +task: global_mmlu_full_tr_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4203e36597dbf5712b8d7e8804697173256f8a20 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_tr_social_sciences_tasks +task: global_mmlu_full_tr_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9cf6352c428dd6fb5d316e46c3d97cfe3db5a16d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_tr_social_sciences_tasks +task: global_mmlu_full_tr_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b86a699b6832e790f89f1cbe6bb563d5a3d09b8c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_tr_social_sciences_tasks +task: global_mmlu_full_tr_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..001cbb288858cd2e34c2ecad4d99811735fe4890 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f1d4e4f156e3a43f128d649df6ce99201a3a022 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/global_mmlu_full_tr_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_tr_humanities_tasks +task: global_mmlu_full_tr_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/tr/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_global_mmlu_full_uk.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_global_mmlu_full_uk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9bbd14cb88ed5aa358aa0dd1502633454034b62e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_global_mmlu_full_uk.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_uk +task: + - global_mmlu_full_uk_stem + - global_mmlu_full_uk_other + - global_mmlu_full_uk_social_sciences + - global_mmlu_full_uk_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3ec01db662d19e88fd639bcfac012aa559c4f3a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_uk_humanities +task: + - global_mmlu_full_uk_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..176b18617e3b55fda06f9edeb57c4237f3b2d465 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_uk_other +task: + - global_mmlu_full_uk_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66b36a60860741d505ec4166590d92b484addba1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_uk_social_sciences +task: + - global_mmlu_full_uk_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4deba6571d2e63b1b7a52dfbc8c17a6cf05e39c1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_global_mmlu_full_uk_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_uk_stem +task: + - global_mmlu_full_uk_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_uk_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_uk_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..5765ce13a8ea0abca99c5f3d10a466956986b3be --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/_uk_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: uk +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce37c7153b358121247314c8a5523b98ab96eeb4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db1433d5cdfbfe1636a8249d7c86d18a28a75c92 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b123ecedf82a4e8b116b4e1d8426db8e2311e14 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..775d2f2c485a36f44dfcceade1349a9d54bcd70e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_uk_other_tasks +task: global_mmlu_full_uk_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f71076dfd6b51d841c44a4a450c20c1635c0301 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_uk_other_tasks +task: global_mmlu_full_uk_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92342ac7c46e7a5aee5c74222570bdc250ce453c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..71384a8b59e22094067a5b9d8b218266347bc851 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6013afe174e9dab8138b6cd653aa7aac98c37337 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27b604915237b4df47792595be16cdf87dc6c32b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87131c250cda56368b1d4595313a9785bdac3b3b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_uk_other_tasks +task: global_mmlu_full_uk_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93109632ccd54e93fad1d9dacfd7e9e6a185a905 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f11fcce9a8323adf61d6ca0643b3f1bfe701cd4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ff9715a0f218bc8cf71a7859db862c8e197e6d6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba92e4b8c750a2024e87fccfcc28de4a54bb344e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_uk_social_sciences_tasks +task: global_mmlu_full_uk_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a1c86ffd86c74ab468774ebd01a9a58f617f4a0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d80cce7c444e66b70af8c3f96bfa6abbea8c80d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f8a40912591a252a67cf28ddaa681e862d4b593 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_uk_humanities_tasks +task: global_mmlu_full_uk_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebd6c2da9942405d628ff522630a0198659306fe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_uk_other_tasks +task: global_mmlu_full_uk_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8b0cf3a819ae6df7a2907b2c2524d92e844de03 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..010dbec34f80dca3dd32a6f346f8f2e11cd19773 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a2701440c005fa4ddfc722b3940aba25b7ae121 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52e80017c6a35cdf66a3d612bd6b36c094d8bf17 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_uk_humanities_tasks +task: global_mmlu_full_uk_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f41dd3d8b4d19ea2010abfaf86d472bc1a614d8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_uk_social_sciences_tasks +task: global_mmlu_full_uk_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72c589ef3c96aef46afa349542cf864dcb704331 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_uk_social_sciences_tasks +task: global_mmlu_full_uk_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e70675d9af32b086dd01a66028253e74ef6dd39f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_uk_social_sciences_tasks +task: global_mmlu_full_uk_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e29c558ea152102cb364877f2c66f968cde29589 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b7354950affa0e6adb65d4033d7d6d171dcbcad --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_uk_social_sciences_tasks +task: global_mmlu_full_uk_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69a03c065d67601ceb8096d844eeee429550f5c2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b02711c212e701d91dfdb63974cb0240f1c6269 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_uk_social_sciences_tasks +task: global_mmlu_full_uk_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60cc0cdda683fbde92229b53b97ebb2e911fd8e2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b62244ebbbfa71e0ae8464e547b5f57e712c9313 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_uk_humanities_tasks +task: global_mmlu_full_uk_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57667edc254ffa8c3b3406f30d2fb891e18377f9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_uk_humanities_tasks +task: global_mmlu_full_uk_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..028048904505c81ad20590187f6733538cd7e594 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_uk_other_tasks +task: global_mmlu_full_uk_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37382bab0038d6a1b29f6484cce4900470589257 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_uk_social_sciences_tasks +task: global_mmlu_full_uk_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d1b046d79cd94474e30b842e71860d04b70e45ab --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_uk_humanities_tasks +task: global_mmlu_full_uk_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12b9da5239e828dcd43a539c508ceda038002bd4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_uk_humanities_tasks +task: global_mmlu_full_uk_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..abb2de2a05e759655b60e1b9577e3626197c2437 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_uk_humanities_tasks +task: global_mmlu_full_uk_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a1a6f34978827b021e901c7b72e6ea27122d1fe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_uk_stem_tasks +task: global_mmlu_full_uk_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec4cb17d3064d4ff3af32334094945d11b0670cd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_uk_other_tasks +task: global_mmlu_full_uk_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afbdaee2e028e4011a26da315de8c7d3eccb774f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_uk_other_tasks +task: global_mmlu_full_uk_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc1fe1bd199203861224ee853cef1e575b79adb6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_uk_other_tasks +task: global_mmlu_full_uk_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f3b18f833d085ffd8b48939433e970b388b18b9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_uk_other_tasks +task: global_mmlu_full_uk_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34b54e34d0f47c22539808a50c2a7aa044403c23 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_uk_humanities_tasks +task: global_mmlu_full_uk_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38706977649f0c27b555b6e4887d9b24ca7ef19d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_uk_humanities_tasks +task: global_mmlu_full_uk_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f9dd1fe8b62a4dfddade286b74f88ca192023db --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_uk_other_tasks +task: global_mmlu_full_uk_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e9810083eccde3b43f2a00df4fbd3e2dc4b2992 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_uk_humanities_tasks +task: global_mmlu_full_uk_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08e3c2af263ecceb300a9f33cde94ea653b0b65a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_uk_humanities_tasks +task: global_mmlu_full_uk_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc02a7b2600f9d736f49e80bb87f9ab8a9d0f027 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_uk_other_tasks +task: global_mmlu_full_uk_professional_accounting