diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..836d577d300acb9020fc49e76d746df5c8006763 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a8e7db926b0f687baf4c6314da45d8aa651d7ec --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4ebc1a5f0f9c86713d68524e448e97d066da95e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7bbf959ccbedb6b2d79322af80db22ee05b14cd0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7faa9d43d9c82f0a35e6831aed097a3d82d44893 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0ca49ae208ad46ed4d23aec781a3adee1e6eb4e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5fd9f0ffeaff1811a9d1934170e4c0a867445d7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..843ea25409dfc301b5420a630d50bfcd9a758e91 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5e56ce044ac3cebff906ee21e65968c480732c7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/global_mmlu_full_hi_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/hi/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_global_mmlu_full_id.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_global_mmlu_full_id.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd29a2ad18ec549f0feddfb493e8bceecd1740f1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_global_mmlu_full_id.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_id +task: + - global_mmlu_full_id_stem + - global_mmlu_full_id_other + - global_mmlu_full_id_social_sciences + - global_mmlu_full_id_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_global_mmlu_full_id_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_global_mmlu_full_id_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9283f55ff308bdb51432c2d300bcc22da015596 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_global_mmlu_full_id_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_id_humanities +task: + - global_mmlu_full_id_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_global_mmlu_full_id_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_global_mmlu_full_id_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74de0f3607f3c32044a4ad5ecb3d67c67cb29c49 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_global_mmlu_full_id_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_id_other +task: + - global_mmlu_full_id_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_global_mmlu_full_id_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_global_mmlu_full_id_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8656b6bedca7e885e01a03871f183c9749665e4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_global_mmlu_full_id_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_id_social_sciences +task: + - global_mmlu_full_id_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_global_mmlu_full_id_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_global_mmlu_full_id_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0e472764f68eb67064c4e11d99b0388e67dec74 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_global_mmlu_full_id_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_id_stem +task: + - global_mmlu_full_id_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_id_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_id_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..198dbd55f5d6846dc5348785c5862e26e9602605 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/_id_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: id +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b18c1cd7f5ebe0a1e22e4172dce1dba6d1f95d73 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65b83d9d750826041b8f0d024daef58d01e5b804 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..11f1047c113fc9d0415c003b35c15c3f246f5b60 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ed992f22d79bbc0e6134cddeda647e9645c1e8c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8baa424fa121882c672c5a8d091268e817cf7a50 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..67b9c935b942d4cdf0ba35264938695061efa321 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3eb5d22804612c2e93a3f580936520c14ce1866d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1462945be52edd5806de1951a35f5285a96c56a5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..980627920d2606e5cb3e2485eafebd7fa7f3dcc9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a2736e891571b829aa205a74b51689df613c8aa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb88c3f8cffd13bb80cb56586f94f44577ccff01 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9764ac3efdd13ec63afd28ee8f90e082e6ad9780 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c70c111caf02dd6276ce1af666f98df440e9e6f0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f82a74b24a4e9c72b097bb8858c675d01536f37 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3cc2dfba0da0b90620b5fbd9624d839dd604fa55 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d511b4baeb397cdbd7840d00c3e461b00b29e0c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c6cef13a7346b7aaf7016cbb4118f9d3df9abee --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e7a44daa636b07870c61ce1639db26422b7b446 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d39c31abb9ce693788fff22dc2d8b86269d6bee3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d92d827af1eceb0648c47e2f36bcc03d9eea7826 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff714ac846b2770c34ae7adbda94e92f65874044 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d5c8141851b15325ed0e89740c277c836aa9593 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ad392b3ecc4fcf43906dbfef56ad504e40b34b0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..850d6d828250476623396242235b809761eb6ca6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1fda5c65ae4b91f0f2c3ded46daeea8d5297cd0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a628ed9a53356ea6adb37bf4e8134458984d8d6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2c447077f9ebd7e127d0c57d601af3fbe0b9234 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75888a3d00e82373b6eec1e63fbe4628c3aec546 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a6ff54ba6b995cf182a942d7c81b5f0b2ecbee3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab2058026d71b55308832cb07155b60b826859db --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dee8c3192c4edb818e01acf2dd4e00f1be27e24 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5474c8bacb189560886b46624c8eb0d37154255e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..464ac67f7f857e02ddd204abdcd3fbea8ab667be --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..518cb30c4e107c0da6da587889d999a4a91c6f42 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90262ada02b54825d78cea5586f5782e396b4615 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8727ab4916766bbafa4de49e1c900faadcdda4d9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da2c8e6c235015ec9f2a044cc836df650c5cc32a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84a30d9d8be85a0c202fbd31ad4573be6b4a759f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fdd340bd88736e9b57544c865286c8b4cf13e29e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..caf3eb0f29f2bfe53f523d7755393e2a2c9d589d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d649fd3e41202ec6bf77bcacd7d388e4a2de55e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0811f1b06f88bd4b8151821fe6a918a4206c52d7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a124ded50f54ef01d737e47b82b54f1cd633ece --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65dfaea7266ee9c31deb2b19413e283b6457ae9a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..804ffc602c974548470d4f80f08a39bc63d001e9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88b37de87a71511f81dad12955392b9249d4dfa8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e851c4929704957fd4d784f4e0902573f8994bd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d45c9517413020f62e8e4dd8f4a751b40afdb2ec --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..965cbad60dfcbb7b8d9fe5cfb856f996512126b7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fdd02d534ab32e2afa85e3ec2621268e98920905 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8d294b27045f1978b60c4149dc95ca9ab12fd33 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f772b0cb978831dddfaf8737b28676bd411e82f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a73d36bfb2a02ec092fadef29175569cb2e88d3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..715e5c31e635458466de832de462b465d64ab044 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59e147aea00069f3138f1ad8c1a561d641755854 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50225ab5f34367f75ede434f7e5f687045f84eb4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0193d12d0bc16e6437b8b105ae6b52af9a42b1fa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/global_mmlu_full_id_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/id/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_global_mmlu_full_ig.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_global_mmlu_full_ig.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5346e1cb49b7065304d4424659a1afbee0b6b17 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_global_mmlu_full_ig.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_ig +task: + - global_mmlu_full_ig_stem + - global_mmlu_full_ig_other + - global_mmlu_full_ig_social_sciences + - global_mmlu_full_ig_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c6ffb6121eefeba7f838bb111e7f019d6038727 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ig_humanities +task: + - global_mmlu_full_ig_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..214efed225785ada97a2a0219c8d11fbcbdca191 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ig_other +task: + - global_mmlu_full_ig_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e27fe1fa179b32b951e30a4f195137a9ee6144d4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ig_social_sciences +task: + - global_mmlu_full_ig_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dd33b62016ebe64174bd8626a9142168e3d7cf3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ig_stem +task: + - global_mmlu_full_ig_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_ig_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_ig_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e876dd0cf7df4d675f29b888d27d7449cdf1a4b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/_ig_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: ig +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1dbf6c8343d742e5bcb8c2db7ba386d82472864d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dc198c25fa1a3241569b6af84f55172ad16d4aa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..078069eb22ea1ffc5ddddb63e8a129511df20c09 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f075e7404f9586fe4db598844e631784f7a4f4ce --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d41779ade0a3af33435afd77c5832eea080a445e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f0e57052c224bf00637d7a40474506d6e31ce29 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78e25dc8d08b54118d5ea0a7adb60e04470f3a36 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9894a454777e0761c57c2d35ee3468a5cccb1a9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8976041fd6471fa8f37675e68411c959457f09a6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5edaf0d5609815ef6d84b422b4ff80ac2f6bc566 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e55c01cbd753200f3e07f3ccedf261160beaa61c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ee7564cbe4b5b4f99424a5b88df70414873ce0d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..555d4fa83db09031ce9961777b4753524b1e5ccd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..783804b889a15aa27c8f642267d21ef6630068c2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..789f95d20a29de34b60228d87934613d7d2fa134 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a5c9d2c5551bd384894e4d5d889d11d21bc04c8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f9e426ca37bb8f3b0687dc58a5b2763bf24dd1c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9b7955c66251e04e5db55bae6927474c3dc065e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..368bc71ddc19dd3702a0fa4bdd29cc0cef712769 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ce77e108b330987057aff3d2cc898aa5fb9b9c1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d859f390f2fd59a9e09c6b608a6aa170ab8cbbef --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29a93f460b4fc085a1919ba23be8d64fed83e726 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74194a44a8f79f4ad414ad2840767d2248cefa2e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd53504d09d28babcb54e238ffaa4b1e82664ae9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30244a64c3355feeecb73a0077d2a857d7b1a75d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..737c0a56c71f6a19b55d72141a2f8f3d815dbc49 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a2220ccbec2817da79f2f42066c1af4f9a704f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7d4c537f50055076f263979c4a3fa5e23852f1c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3051f013f1ca6d6aee0714e5722fc055a1b1697 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d48410323b57abe316a6ac7beb9f5e331d93c932 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61e124fcfc8810f3023f59bb180997052ced4ee6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d83a63d94732743d85ff87aee3f31aaa96d52eb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..787e3151fdb9a57b3c4e92e32a04930c2319072d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c61845911f8ad941f86774f687b17a2abaa6c9e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a8511d215406b568862c08e3e9f5d05effb6bb5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46254ea14c3636103dd882d2f61d666809725b39 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bce7502915d3774927d494fabf2338f5e7c22eb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93c87fbe662456de254a891f330e5b83490d2008 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..780e1c894ac9987e841e9976cc1245e481636878 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d30ece9a83aa81275a15b79766f25dc8979925e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cac197c78e0d813636a24dcfa3adf2d36f360292 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a38245103d61b773987ccdb1ffc2235704adb796 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc545d84ceff698ad94e4da7ebcf114f0d3b8a44 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60ad22feb3a862a7fbda02ed3e8b3e916c27e8ac --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3cc556070e3aa52a43953dba17d58e719f5cdeb8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f6556324598955b38ba5bd6d516a2aa5975daa7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db4affcff562cc7f26b54ebc6504966ec7958898 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18d3577349f279b26f95790282d1eac115e1aee7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9db41d007aa732ddace06d4a43befb8c93fe8a8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7fa28b603d3e7617b207cccd464c83007b9b4a02 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..639be38138523058e2ca9756ecc97209eae07eeb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d31af09f4db98c7826584f80a25494ea75806eec --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..200db46b00cd5e83afa3a136c08a03438f36309e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65a3e4e1a85bb47e638b3ae42358f37ba9e9e17a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff0b0505f52ec9435bd5dcd5f9e63ac6ab0317bc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b437c82f31a01686ddda4e25b14f28e7bb24a8fc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fbc7cfd8237240d2596901daf74f316f0ba5cc6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/global_mmlu_full_ig_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ig/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_global_mmlu_full_it.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_global_mmlu_full_it.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3bcd69418c599562fb87eabcd6635ed286fc02b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_global_mmlu_full_it.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_it +task: + - global_mmlu_full_it_stem + - global_mmlu_full_it_other + - global_mmlu_full_it_social_sciences + - global_mmlu_full_it_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_global_mmlu_full_it_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_global_mmlu_full_it_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d072ccc57e1c027d5b6038eb94715f01c916931 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_global_mmlu_full_it_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_it_humanities +task: + - global_mmlu_full_it_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_global_mmlu_full_it_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_global_mmlu_full_it_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99fe18cda989217594ddd42883c3c0b2870f29fe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_global_mmlu_full_it_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_it_other +task: + - global_mmlu_full_it_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_global_mmlu_full_it_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_global_mmlu_full_it_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15a457a5592ab4c8a38c67c828e98c67896a3a48 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_global_mmlu_full_it_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_it_social_sciences +task: + - global_mmlu_full_it_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_global_mmlu_full_it_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_global_mmlu_full_it_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf7a555dda5f3a6a88e50703448b4063fdeeb99f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_global_mmlu_full_it_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_it_stem +task: + - global_mmlu_full_it_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_it_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_it_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..a74fd7ca2a77071ee02bf46c67932207567f9917 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/_it_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: it +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7351c1eea56ad7d91eef0bc87141d7da1cd0004 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..436cd3f661051384d834f12675aa399aa069052e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f98f0f20edd395b148f5d5f39bb60bb9f7f1d796 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9d931feddf665778f78091cfd672f7c0a4c013a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe4290240ec5ffec9c0d0f95804efa55c595b00e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..71b8f45e2b8368c0be2fe2a6d8b26f5bc30ff95a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d29bd758518c8ddc7a412f4abbcc613ba5e0fe8d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f740d259df43bea0ce866be2340e88fa96151b75 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7568fb7e67f44f4755c81dff9cc008807a956ee7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9bfc5ac1ba55fed2628a9702b544a1288277b444 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2101847ec748fff0cd4c5dd694a12b38a9a3b468 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70b31f9a81bd66aba60f539c4891e570c8816237 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8917d409237b260f8e8428e56cff1629db8e041 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a49352fb72ab2b57424961b5348f29cfe2e3889c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27f0c6c330cba260e88a2d9bc8a4aade6c13357c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd78a52e84b734b46e754eab94dae7a5f32a7ccc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8171fcf18276aa2fa1c0bdfcabc19b639654ee89 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a952ed444b63398acc137d3ecce77de89fbb21d4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..939ba7525bb022b60211221597c1ca3eeadf0223 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4524d4dc3259c52e9f7cd0e339489529781c315e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dfb1649fc16f0eeec93994333264e84f2a3f3fe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..556aaf20a319458b6e4d3a4bdd13868af84b61dd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c1d5b6008944c8fc498a1b873b8f7ff87e8579c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a35b6bac89dffb1337f7782e91a0119d288178e1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74c01ccd4481afc27dbc485cb5b09b7cfff43077 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6bec02c4ca085d2e6529c0fa0ab7ae57fc1d06f7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..551a0f8da6f8e35b523756572f73bf7a2e0bb27e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3cf7144bdda5e118f79bd78b683eab29df2c8325 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17088e513c2d51ec3ee51f3cd4713a76bdb05440 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3f35f991c13b7243175f44ddc87f7c1c3c4dcbf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af222877bc8b043b59d88431074038a8b136b0d7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..698ddb5fc3913b21ad05a94c5e875179b44e0698 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ff49730e275d7a68041039ace5f155b804ea8f2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58d32fa4afda0cf544b6da454cfbf508dc4af253 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7c47e55d4f495dea81647b1b4a460282706de77 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e100c0e9255e85c15f3fc87267f1228a74f06ddd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a07444a85b407c3acac51913c97eaf604bfa5d7b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfd3b7a5ab60d33fd4dad138a5ea501f3538ff15 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b5feeacfc53a4bf7df581d5a7907344ba52450e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d50b46f44e9fa6b7d5287143e01b05c07b800ea7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b02316caa27431e6cb90aec3cfea45efc89c85d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b638b50de1cbcc90c682eb0017fd89f74df2a040 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..520a8beaf092facdd2bb080addebf312d9d952aa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..abfc739509650b46ab19238d23e131f8bef6daf9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cac74152758293800245078fe7168d680c213d41 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1d94976dcda73baf2bab8d6e910c5d97062d415 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74bdec821927ce39c943da94f5a7f4c5d6e1bb06 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..acf999a2947ad8231d79399cc24422f7e6b8b851 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ec4f58e3675b456e3ebedaa36c49ce45f9a4560 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b53cdacb8fecbd3860503d04ea8fdb78f86c860 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b7a24f390d50da9ca46270f5a156f6b7c6f18d6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..727cf4b6e30a1c91484ff8afdeb6a30f04af79bf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90fd186c56ea787eb26099c0e58950d79472ed28 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbc77935aa4d4692a638506b2cd7faae8441f391 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2a923b6b2a3f3abf9ed04d6538f4aa62845dd0c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72758a56c4df5f38c667ed0ccd2fe348a24a8ba7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4491c51bc8e3e781c2df0fe1902a13993d9afd3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/global_mmlu_full_it_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/it/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_global_mmlu_full_ja.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_global_mmlu_full_ja.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af25573dfe8cfe351e2e7cda538cf4129b9ca1ef --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_global_mmlu_full_ja.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_ja +task: + - global_mmlu_full_ja_stem + - global_mmlu_full_ja_other + - global_mmlu_full_ja_social_sciences + - global_mmlu_full_ja_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a063eb0b663cacbba1b79afc5502efe5e5681ee6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ja_humanities +task: + - global_mmlu_full_ja_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f9b95ed6da3974a58fbd81cd0b664271ea1a022 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ja_other +task: + - global_mmlu_full_ja_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4207fea43fd0cdace8edf8182bcac5b57d1e6b24 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ja_social_sciences +task: + - global_mmlu_full_ja_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ca6ed1c06a4fd65c075816082fc9bf2abd5b0f7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ja_stem +task: + - global_mmlu_full_ja_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_ja_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_ja_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..a31183b5d7b0b6896eb23d8117d8c7ef6420838c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/_ja_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: ja +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b65a75be2476d4dc8bc1244d7b26f8c04f8ad99 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e735aa349df3f7c8bfd787044b2061add9057093 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0ba89475b74e0f3d4492db08baefc458bd9656c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c39d286c7ffba693f61da3749be7bc43b03d858e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27d09b8887a4138f9385c89b8e4ba57cd24979c8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15e26a51a3ba91a7397adf477b151d7043a8841a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52c92423e7159ddeed225b000a50a1cb28ec8236 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a91a7d61717a3c2ca053a826efc3026234aab59f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..67dcd2a02289efdd3cbd150bacb2f43d2cd83d25 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c55ab2a25dee6fa229f5488a423cf276ed981291 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5413c86d5602d40aa982466a1dbbdda5a5589e10 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..276f214e794fcd316ec34659bd0082d0b5d491e9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f823ac44b333e51e4570ea77784cd02818d0a4ee --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbc6846a790eba71d2f5b6d763a4f0fd75c5bc49 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba7295759a40b51bf0d066b8081258f7a87a24d3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..138071048c21e33f974483b47ddab8eefe7be743 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d88d5685abdca8a4bc9a36b4759820fd79b6a922 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64cb2b9ee4f6840e17722208a2cb07f65d6a7e75 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b88adf90587b18715066659e4c692c4017188e10 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eef67cc7773890154ae12dfe0d716c3883bab38b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c90e5fb76b61a6a581b6526bb448b3c8a678747a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8318099ac99a420effc370650e14441de386fb77 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ed5a6209bb1330861a4da399a63a4c87696901c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ec0ab8422873fbdebd63e138ea80294c747dab3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bdd9555d65c540bd4a6277853b7b8b2c6c407ca --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3abfd81b1ca3bb9a7e7833a10d8d67535b24a150 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..483161a60914cab158c8b824567d25261701aaf8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..702092af446264025bc997c8b8ceca537626df03 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b7ce92eece5a40aa9ef4bd2e10a9f970cba3362 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c68acb8e827ea1dc1415d8ddaed18ccb5b34b197 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1b91833201575b2dd8f1a049a04a1bc5826c208 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dee1f89e90044d9cec96388b8477ec723b1622b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3612a7eec71d04f51167b53c0d230fda75bbae37 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b70204fbcd7d0e89a99a532ab220f19151be54bd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77ed3c97d7e0d49191119fd8158005e84e5d29a5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8fbb261f7e6ac1b4e3785e466c29bbe7b4d6446 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58d4afccbebf5c4150409d701159ab2d4a30f8b3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e664390f0bf96f91706a04f6b89edf2dd761bef6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf495ae2272d991ad46dc0b15e4cc4956bad00ac --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1349771effd8889c35bf139f78d1a82913f149a4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b513ac4f22e577808d87f45140a3876f3dbdf14 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81659bf774474bbeab8a6ef4f30030da1155ed82 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e77694bdac7e0ffef6457e1e0ad2cd0a3673cb8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f322376dae0acca5b0c32bd649545ba7d225f35a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d58fb0e60c1990541e5b2eea85d74aef9c23b66 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..238653619ffdfc30b0b61435434faff5fda992b5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a044bf99c52e926965669c9341991a0dcc9f09ef --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b828e0e28bd24315cb2b295cd96463bcf22ba930 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7aafb6c4cc3d41d8d9e005f67f749d0964a56530 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0cf9905e29139af85f77696a79e20f768a1d183 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5ef36c18369bffbd5d74a457398b28fc5142ae0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..565439e6d7f3ef9380452f7f946813a4ec417972 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7d21bd64a5770ad38531599a3b0aae3ca33b120 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cc44c1c12a4e077cef78f08fc7815067a0d3190 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ebdb14a8c2a008b610419ee51476d8cb0402b75 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6f833672faa6d6cc43225f43cae9089c0c1cbd4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23e66e063640e906f4c31f7fded8b0ba7fbfad08 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/global_mmlu_full_ja_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ja/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_global_mmlu_full_ko.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_global_mmlu_full_ko.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b9b419701bee34d47215496fd20cd53a95f4038 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_global_mmlu_full_ko.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_ko +task: + - global_mmlu_full_ko_stem + - global_mmlu_full_ko_other + - global_mmlu_full_ko_social_sciences + - global_mmlu_full_ko_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7690643456b274d6a2f07a4693ceb2371eae4e1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ko_humanities +task: + - global_mmlu_full_ko_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8990ae957ca2bf03544f30495f92d95d641e0e70 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ko_other +task: + - global_mmlu_full_ko_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0bbfad7f24acb3042b85c0f8825f51f6e298a6eb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ko_social_sciences +task: + - global_mmlu_full_ko_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18b7f17bae7d6a67d027342661184672a0b95a32 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ko_stem +task: + - global_mmlu_full_ko_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_ko_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_ko_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..85c72001af3d11ccdb4042cff8c496fedf2ce1c5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/_ko_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: ko +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5959d7881bc0366bcabb2c0144f761d8c4858996 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebb90860fe61eea9a47a5f6b0561b09a3d194338 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..670846b327253b15226b264abfc3ad5f76e46d00 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a44e430211b6f47e042802ebd01d7b9ca5d7c04 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9e29697c08bf5e7be4aa89032581826e24fdc10 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc3644680d3e97f82d4fbc801fa0feca11136f68 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2eb0f416f3f3b0db9da85a702cd0df96bc30fbfb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..044f1eff5a42751ada4ff2d05feba727ea68415e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9929097ce772a874064b4376f4556b446b048a86 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b78c24e6fa2419591a3c072b19dd7133a168aaa2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20c3fb20a57b1cbdc87e15aa8016e01632135551 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f95457249b51a31d715ed22a9b39a37d39ee93f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f79989750da6c136e7971c6afe2f0c7c5febaea0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79c35ed7f98c8cd53855ccbcab54b64c62e0ce54 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1444a2495a807e9fc826d2b2f6624fd90956bb35 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8bec91b9913b00714dc15978b6f4dd69c546099a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cf3109238c6074386ea0d9a30abdd3b489661bb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a5f7bd5403deb9e6ea8515c9c4e7c671f120635 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bdaed574d0b464fdc393f49be834a5fdb06375fb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..193a064c984f847b6dc679cffa6cc0223fa30738 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d2ad648afc5763de6fb05e614043a61939a3f0c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a48b602dd0b762a16bac7801a26c8cef0b819fad --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc9c20eb6a430de9c445b55a7310870948c6737c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e86a27fa5c2f44f2cbcbafbf9680a0032bb91016 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b947f7f3f451ac902ca4e9d5ec036efccf7b81c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9184ad9ced39c8a52377da4eab6b856875d8fa6f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50b6a150332ca02078913fe01226fc1341944ef5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..974e3b03da1722e347b4e90c18762232238a830f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e617e8cdd6144066c57ece957ca27f0e1493ffba --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a0105961c9365ad342dbc006c8425ac868a67f5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a696675df6951a6d882b89e6c0da31fa7fe78c82 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eca86cbe496ee634f735dd2d6b2daf3557320f7f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69e3a2df40e209b8105e28c08c1eb63656fb4917 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed3e99fc51a7d31798eca2b3b18487fd92d4866f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..651f389c20d99fd93cab22351d71ca13ab890f87 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..001807ebcabd19691bfc34597eb9d0528a7a549f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01eec477c5c460e6334255f3463124604fd0e174 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1126c6b7ec6683b245df8f76d16bef755640eed --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b8332705879711a026ea9df177fdb51f8bad090 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3cce25c098de00ed09758aa874658822776f0b7c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65df17861b5182a5c75e90fde31646dd068a37ca --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04b71e2a702f13002c84198f4d531ab07fe91d45 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f1e7fa1bffacd485db4c9d5c556e331c63c7abe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c657543aa52eb0e75a7d2a64fa959a02c3a3a478 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dff6450f921d469568cc3f1d3e7655e090b2bd0f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21f058af0efa215550dd81539e55c44fbe63fb6a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56aedae91fa5a7bc214dbb58c2ad3eae52694b33 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24f83b232ec63d61785c9a4487d597295702800b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ece9dc5c6ee09e19f6d02523b1a38feaa62f06c9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..439309570bb0360304ddc46e39356cdb8ebfab7c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98ff6520d1b3037f8184afd6da45b1b8ac2ab233 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a5b07f7e31cdac2c3417e9ae1c6de2bdf46bd42 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3663391a8fff81cf98712224e039b83b389caa85 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..902b4443e2c696a992c8d4089604e7da23adeb3b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36e1794ceb2bbb5737717d7cb8bbc2084d1e861f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64b58d6a95907577fd3f072cb0348bca59c80656 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7289671f5c4f6942fa770a835fddd60f3ab4fade --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/global_mmlu_full_ko_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ko/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_global_mmlu_full_ky.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_global_mmlu_full_ky.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33b2a4df9ebc7f8897d623ac2dcd20670fc131f1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_global_mmlu_full_ky.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_ky +task: + - global_mmlu_full_ky_stem + - global_mmlu_full_ky_other + - global_mmlu_full_ky_social_sciences + - global_mmlu_full_ky_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e0368c2b6c777c29b37518cf7f574ae06a703d3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ky_humanities +task: + - global_mmlu_full_ky_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1bfc89ab6777fbf1380bc4ec506ac2e6ccaadd32 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ky_other +task: + - global_mmlu_full_ky_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ae756c4bc364d0aef2a8b093b3b27f1d603b817 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ky_social_sciences +task: + - global_mmlu_full_ky_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..817456fcb2c31df7614c2dc8973b9aac83ed92c5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ky_stem +task: + - global_mmlu_full_ky_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_ky_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_ky_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..1860cf33ec81afebe9a52e57f8174a1370bd33cc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/_ky_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: ky +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21338a569ce25b419e959f73732269d734a19d8e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df2635483c6c407c8756bd25151143f87fece8c6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e0f6abab0550e86425933ae919f5f185359c2ba --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17656dc6362b31b43686f1d0c6640219365a0854 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c053b884c0029da3bb76dc1ca3f00c9a0df0383 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3649210635296607da1a355402b61731c1d064de --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb9f858681b003e4cc7670866ee8c99dedb63e3c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4b15b5401fb69a53a16b73369019578d4c656b3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5657b66bce4b9108fff6a0e851c827ea1e3023a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1e0c25e6c4f9685fb37b4aa4446615714f15f04 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fac1d80f31fe382a056a9a2be7d60208c7d413bd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e35718d028c1bf3aa1a1be071e5441ff42293b0f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f165ec61962550c284827e68cc905d22f31568ee --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48670c7f8d17747e03a11603c61f563ca717fba9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29d241428f1984fabb2071cd248ead66bcd99ff7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9aa7f81b48588e4c43d03108a5af42ab13c431ac --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70a5bd8603a71d3f9d57ca9265f68d1410bc70cc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f678c0d130bd4de90abb3016fbdbc18e7384eab7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..750bc68b8c8c769f35832e610a1f8c11d9cf15a4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7700e37fa9d1118af9916cfabf9564c44e07894d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c805fc4e91c7d29dbb2bd33ecfea8c8c116639cf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01c67f8e332cd612a810f4801e3cb16211e7eb97 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ccc5c8b73d1d1e70ad5c12504acf77fcd15c0273 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02ea66ef6b4be7105afb290010e535be8244e7d2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f693296dbef784f629fabb544cc95ba3f7dd4630 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b05e279925c8fc21efd67700618d830a5eea9ddf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d596290fa3ad19f7641a0dd178291f00e21d341c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f71865c250b4424f3b4800b19ee176adcb06d4b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..635873a1f1598a3714c7b5009a05750ef3e26039 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df8cfefbcb7d17314d05930ee8fcd140b2235025 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c75f534138a80453eee56e52a5da632cc06835d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e73edcbea8ae601b1fdd24b8b0dea1d9a4222d1a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4e662a56d1752251a61149728c59059c6591925 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c2556da1976fd1cd586ecc1a1d72be01fc84caf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2af16190992c5f22249fd716fb3a029e0fba1aa8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0994cc2d715489ce4cca164cc2ae19902e251ab --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7933a77b34683a2f9d3d2557f1e13fd23f1c072 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6e525a18a939d63290ae9830a78b8f840ae5388 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03f70aa02dd866cffc52849bbe0771933557f10c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72ced798867ea4f86e9d7bf0717608be99da70c9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..371e4b2176d9396677782c888a954a1acae2c1e8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e693ab8b9f1a4c5e8d149704a0bdb102c7692916 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ccafcb1e0e6f06576f38b0b4e38cba94fbc1abe7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16c19b29eb879b3fedb7f9f8411d9e9f918bb109 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6c00cb0e7cc9a906dd01a4c25e663d9cf09908e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d6d242b0181a2b7039f5998d6a02108a90bf585 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ff2e08d4ae0c4291b663320d9ccfc918bf6f8ce --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37c6a892286f4ce10cb682e60829b525b33fe54d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b4fea0e5e3b23b9b3623e7be6fa79b98db69592 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a6ef0b531d1a185be293f35b82c18340e39577f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dce1b6d2d626b35d3c497e9a6e8289017348c39a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..168cae74453a423c475b4ed7f13f0fc2f7b9cda0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e24b8167a177e2a5cbae1dee58e34ec2eac20a8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d1ad959349ad14484c27b28a419b3a1081c407a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36cd7e20620780639fe050aff5facfc45dbade27 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2a7791512f829437488c5fd8117acc96f75c618 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..563c1397689df77649e0a29c291ec3009d9d2210 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/global_mmlu_full_ky_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/ky/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_global_mmlu_full_lt.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_global_mmlu_full_lt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd2cd37d3ee9e4c7a1efc0862068e4444f6531f9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_global_mmlu_full_lt.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_lt +task: + - global_mmlu_full_lt_stem + - global_mmlu_full_lt_other + - global_mmlu_full_lt_social_sciences + - global_mmlu_full_lt_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48ad351f725275c8bafa0cf32b44cd3ccb2cf134 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_lt_humanities +task: + - global_mmlu_full_lt_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f63c35ad2223dce8e1edfd1b4e6f4fd2539c9db --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_lt_other +task: + - global_mmlu_full_lt_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ababd6d14e6b2745ceac16953b93297bebc06c8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_lt_social_sciences +task: + - global_mmlu_full_lt_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a59e683357b1994872a4ee00e498cc413233496 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_lt_stem +task: + - global_mmlu_full_lt_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_lt_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_lt_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..edffa070f41fa97709a3e83fcc2c9d3657282052 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/lt/_lt_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereLabs/Global-MMLU +dataset_name: lt +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0