diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7090a6e1d86783c40c01b0128f73bcb29cf63d1c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_uk_humanities_tasks +task: global_mmlu_full_uk_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b43dcfbc70e0cd14292003d7d1b5f61cdd97e1e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_uk_other_tasks +task: global_mmlu_full_uk_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b279a94c4a561588539de397fb880407cf22e050 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_uk_social_sciences_tasks +task: global_mmlu_full_uk_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b45dc621ba49ba94880829374c9881d3fbd3bcf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_uk_social_sciences_tasks +task: global_mmlu_full_uk_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ea308da21fc68715e6fa7f6b38e2adba20c547e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_uk_social_sciences_tasks +task: global_mmlu_full_uk_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7aa08ec041e1faac157d3d686267e68ee4fe1c7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_uk_social_sciences_tasks +task: global_mmlu_full_uk_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d089e77838514827f7e062de041e112e080099f3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_uk_social_sciences_tasks +task: global_mmlu_full_uk_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41b627f5a1fb910a51519de3bafb2409e061ae7e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_uk_other_tasks +task: global_mmlu_full_uk_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5d6d4151e5d1f7aa7b0ade57c888865df4ca8e2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/global_mmlu_full_uk_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _uk_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_uk_humanities_tasks +task: global_mmlu_full_uk_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/uk/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_global_mmlu_full_vi.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_global_mmlu_full_vi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..278884741d6639dfda1072ea0564baf63937dfc6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_global_mmlu_full_vi.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_vi +task: + - global_mmlu_full_vi_stem + - global_mmlu_full_vi_other + - global_mmlu_full_vi_social_sciences + - global_mmlu_full_vi_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a05accaf43fc09f9ab484049e5dc77f64a9e9f9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_vi_humanities +task: + - global_mmlu_full_vi_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..880bab9a3ba2ab786ef6489b3fc60d41d4f4eea3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_vi_other +task: + - global_mmlu_full_vi_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6da224f2a0a4db0ed65af52ac4282f7fa41120a5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_vi_social_sciences +task: + - global_mmlu_full_vi_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12526ce7a15b5c02b79925b0c89f2c0bbc9ecb5c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_global_mmlu_full_vi_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_vi_stem +task: + - global_mmlu_full_vi_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_vi_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_vi_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a0ca81751e7eaaa78e349f2a4764870a7984768 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/_vi_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: vi +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47dc80cee15f1aa97ab49861e68f8d5a498f87b6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d29cb5836b46e19ed2bbebfbd05f401e55981ffa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e3ba1dc2c6f69bbbbf30e8fdb02a0fdc9c88300 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3afecdc13996982b2a46c8dfd76583101368feb6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_vi_other_tasks +task: global_mmlu_full_vi_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34a90a8edec01226b73e9dba20f4f47edc04ff4b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_vi_other_tasks +task: global_mmlu_full_vi_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63a4c772641b6f99ede003b18fe17654aa1edc18 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7226e020505a05e1a2a0c595835762198ee89ed --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90a9e0b2c36f6ed0d3689d23f9322a6a02531966 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a09173d6ad778de596aa464ed87d587820244b8c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22dc78bdf926614f460832ed79ac4b4f38351af4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_vi_other_tasks +task: global_mmlu_full_vi_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6f8dbca3b04ea56b0841e1094423bdc6f6691aa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d4b3d605fe5b3d4fed95add8207e75ed108de26 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c501d0a3addc3293c9c896ab3bcdce6b5260e11 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0936b3be7e0c07df5fa454e790e91b2443a5757 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_vi_social_sciences_tasks +task: global_mmlu_full_vi_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b23387f8f872149fc95b68cf4a20e7c539bc4fe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c09826634d38488a073aefa59deac43bc9e37ed --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21a28bb4bda7edda1318f3b07765b6f6c7ac1a96 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_vi_humanities_tasks +task: global_mmlu_full_vi_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a912dba1ba51e7054c59a220581b3658e41c43b8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_vi_other_tasks +task: global_mmlu_full_vi_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e334fb1c9df66b65b35d6f707ba5588a1a26c341 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba98297eb3846b0ebc5d9308a601ba4d89ea3eb6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22e0b00a3eb94bf1ca3f854282c90816457e502e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06507b7ccccbc9ea567d832cd612702646bf18ff --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_vi_humanities_tasks +task: global_mmlu_full_vi_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6eeec7ad17f7131ff7d017b88a27caca51e268b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_vi_social_sciences_tasks +task: global_mmlu_full_vi_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2faf2b09e256fb3a21ee080dccd307fac737d69e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_vi_social_sciences_tasks +task: global_mmlu_full_vi_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16ed50b8c79fb2fec977084c3b862fac8cc1ae8c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_vi_social_sciences_tasks +task: global_mmlu_full_vi_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cad75ecb6cdfd47e64864657451ae0c9c83d516 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4499711fda825f3aae6c70abd0ae5a048cb112fc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_vi_social_sciences_tasks +task: global_mmlu_full_vi_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb92f446d6345e7df44291b4d1fccd589f07b163 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a12e4debebefda50729ed3302c62ee41c8b5d2d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_vi_social_sciences_tasks +task: global_mmlu_full_vi_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ae34e4de18fde6cc7dd1c63542409c403ab4659 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ad96b12da739b85cd5a59a1e400d403072224d5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_vi_humanities_tasks +task: global_mmlu_full_vi_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5df3661c7176e254381ecff370b195370ef7bc35 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_vi_humanities_tasks +task: global_mmlu_full_vi_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57820fab48fa914ba683990f6b8bc66ce896ed9a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_vi_other_tasks +task: global_mmlu_full_vi_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b53962b1354a3467b7df3cab38b1193fad7d62d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_vi_social_sciences_tasks +task: global_mmlu_full_vi_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f81b09e5bde001c714f25d3ffc059901cb4099a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_vi_humanities_tasks +task: global_mmlu_full_vi_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52ec47d4fc75c827612fb61d7df981a6a340301e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_vi_humanities_tasks +task: global_mmlu_full_vi_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed89994d5ac60fd1f1ab98a812ae1924fe5e67bb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_vi_humanities_tasks +task: global_mmlu_full_vi_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..258bd8c462d1eb4a947ff3b8d514d46bb2c6412b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_vi_stem_tasks +task: global_mmlu_full_vi_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1bd2f606e088fbfdc421b6a6909658d1ad6b7a7c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_vi_other_tasks +task: global_mmlu_full_vi_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..951a36422420f7d229f97b7105908722a0ee0f23 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_vi_other_tasks +task: global_mmlu_full_vi_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d6060070171868fa9c0531db68bb39ade3c7893 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_vi_other_tasks +task: global_mmlu_full_vi_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0cae1b03d6d0b34ffb85fadf77d2689e0ce2c08 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_vi_other_tasks +task: global_mmlu_full_vi_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07987487cf71c50ce6081cb2da22c7eab8d0915f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_vi_humanities_tasks +task: global_mmlu_full_vi_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a852bc65b82316f7f26e09658c40e1b2cf73156 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_vi_humanities_tasks +task: global_mmlu_full_vi_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..42b198f3f17690131b9d876f2454070d5e08d82b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_vi_other_tasks +task: global_mmlu_full_vi_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7ffc31613159614ccdf0921cf808bdd196135d2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_vi_humanities_tasks +task: global_mmlu_full_vi_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..963496749c60124491b3465eeb0278eedc6ab7b4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_vi_humanities_tasks +task: global_mmlu_full_vi_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da949e34f219cc1acd6b5b7a6a30c9c6159e71c0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_vi_other_tasks +task: global_mmlu_full_vi_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81c74535e20f83e5eeab83b9412dbce07e6b0473 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_vi_humanities_tasks +task: global_mmlu_full_vi_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7315b35353d98f77fa9507997f587d7510e5e212 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_vi_other_tasks +task: global_mmlu_full_vi_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2eb16528320e8da3a4e680f01cc3182eb57f331 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_vi_social_sciences_tasks +task: global_mmlu_full_vi_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12933f08e0004e269019e5a423914b6b48e6ecad --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_vi_social_sciences_tasks +task: global_mmlu_full_vi_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e90ba55470a49b20b3d90db0116d395fba9d59a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_vi_social_sciences_tasks +task: global_mmlu_full_vi_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..056c757b8288226ec5d0d683b403f389e8b2944f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_vi_social_sciences_tasks +task: global_mmlu_full_vi_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bcd95d6162d92e6d6d8d7800c5f89623355e8a9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_vi_social_sciences_tasks +task: global_mmlu_full_vi_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..775b0cca8ddd581b43d04057c4fc39ea59e96cf3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_vi_other_tasks +task: global_mmlu_full_vi_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db6ba6e02682aa894db8a1f77528eba609813816 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/global_mmlu_full_vi_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _vi_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_vi_humanities_tasks +task: global_mmlu_full_vi_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/vi/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_global_mmlu_full_yo.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_global_mmlu_full_yo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef4930b6e4cdfc23f246ab5301cdd9f8a115197a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_global_mmlu_full_yo.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_yo +task: + - global_mmlu_full_yo_stem + - global_mmlu_full_yo_other + - global_mmlu_full_yo_social_sciences + - global_mmlu_full_yo_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e3b3c119091fe9232626186469b3cffaeb308a7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_yo_humanities +task: + - global_mmlu_full_yo_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed81bdfec55a90fa017c11850ebeb51e7058c970 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_yo_other +task: + - global_mmlu_full_yo_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bab52fa22ac84fe7157b4027c86732f5e4d15c23 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_yo_social_sciences +task: + - global_mmlu_full_yo_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3687d569e9243d3ce45d662e45c97ea83c3aef0f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_global_mmlu_full_yo_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_yo_stem +task: + - global_mmlu_full_yo_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_yo_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_yo_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..ceefadf5b13c04af262f3ec7cbabcf5e5dcaef58 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/_yo_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: yo +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef817a386a77acac262b31575a15d737d85da090 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3bae5d55846ecf3f69599084b8e610233105d92 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b39aa143c866aaafd07f9d0d5ca06dc7ef36266a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..588329820c3a9767063e57c63a0b47526e144ff0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21dcf8421b0938cc1e87f8c961ac4b67a9f57b76 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3abaf24c686ee6e13a635044ef62b9c919eeda6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0468634b099bac3e0aa9e86a3294fe0839915e20 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df6e5844e7e96556d2683a5fe1b6a1cee056c960 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0542a4fe21ad3d91126287fe187e9788846c2860 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cce0b497d7c42f8e6c5090dc970f417e1b0f2a83 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84ca1413e609719b95155921f913b623e081bdc9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..001689e934d2ac260f1cc99af241ec86c5f4d21f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcff962c39f1ea58d2f9801bc7d530c9c32a6f23 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d055d6d5d84e0138ef0c6d5e26f24ba3ef46198 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c21f7f0220ecfa1f17559e620db85bad7e601899 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b6173f0d1ed576e3e83df8acd96aa5ff5c25a71 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ffc9740c51e41201bf3e08095c6dbfc4fe42e8d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..394a143ac372eb79debb4563cd8aabcf266ed6ea --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0de18874d4dd74e783688f1b463acca588f8b98 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02b16faec059089214d6ed9afff85bec522fe2e2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94733faa00ce0a4d64fa224127d043cd65ced627 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ec4070e8e086f68714c743426a69f0b6d4769e1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ab051d9b2178da41229d933d28e339ac41af040 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bedf7f204873c9cebdae6f6acb00f4bbfc80e03c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb4867094d9ad787892c1b0dfb45c05074238fff --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cea21a893c6fdb26def6f68c3528c72a63177d30 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8eae6cdde52619df8084eca39e89209fd66bc2b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cdaca54fd1f2bd6989adef5f30f6a2cb08e96c48 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef3d75271e5f2438315d1678cabcd5b8cdc58052 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ec62db01f62cc47d12566beb06500ca56dfa9fd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c8573c4f8e397f934e3b13d56e0c2036df69d1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52f91d43bfaf7af5c95393f34dbf8468e8948f90 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ab0ec2b9ba72b1ed9175ed96f663a60cb977fd3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f510c2d1058c90734f1c5d79a6818ba7922a1f90 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b657110abc7b2754ab10ce098ba0975c9b91e56 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3ac0a521fa21804d313a28dfb91ffb30bdc3bf3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7a9e71860cb9b6eab9af1d0ca653a7d04883269 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a61d3ae9fa1e2f5d2542fb1c6dfb92dfb1a3b1b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92b0b526de32184043682d990d6d612abbc1ab43 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74c17559dbf18b6afe5e0e22da74e7f6c55eef9e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfc2c8cbcc33ca978e2f20cd8deacd40c23cf6e1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad12bde64e609adcf43a18057dd13dd95d79ffd6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e85331c907b1ee1fa21beaebdae22ada5a5df58 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a6a6fc6db60a49f2a3228fb4ade06b4f25ab0f1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62d9ae7bbc09fc99ba608e4b6d8ae0b520ffe5ee --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de42ec7a8f8655372f5ce2efece4105f854cd0e8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2ad3236230f03f7395a195d5cb9a95f6dbff054 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..198f227b893a3758d463ac50d867e6ee3355b76c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5942f74b5f036208a319abd9b37b937b2c02980 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..efd4ab7df34d4475735fafa7876901ecfd429cff --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1956c87cdf7676dad84c956e5d32ace13c39841 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c6c2b8c1c5444066af0541751816b9198caa307 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a12c4abd3e44cc1e7b256efc95bdcf59ec2a6308 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5747900bca8d2a1c00ae902353f5d1938e03617 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..493dda39641e29de5d42257e4837dd196aeed92d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..420b1b01c8bbe069eb3b397b29e8a3ffef620a21 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0964b30717ccf68ee13a8be218f52cdbf56d078 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/global_mmlu_full_yo_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/yo/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_global_mmlu_full_zh.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_global_mmlu_full_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..926c8f88e7a0e92e43e3b96b44413205fb3e77f8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_global_mmlu_full_zh.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_zh +task: + - global_mmlu_full_zh_stem + - global_mmlu_full_zh_other + - global_mmlu_full_zh_social_sciences + - global_mmlu_full_zh_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_humanities.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb347da8f853d37744910ef81a62421128e50cea --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_zh_humanities +task: + - global_mmlu_full_zh_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_other.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98d4ed5e613fee710e2f3fe210c4da2abddd7e9e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_zh_other +task: + - global_mmlu_full_zh_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_social_sciences.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..235012e6c715579dade6820e7173e6fbb69a7f47 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_zh_social_sciences +task: + - global_mmlu_full_zh_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_stem.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..660486a4f41437936635eb61999a13c457bc2ab7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_zh_stem +task: + - global_mmlu_full_zh_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_zh_template_yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_zh_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c83d4957bae4fa07bf63d4a177331fd2954c37c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/_zh_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: zh +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_abstract_algebra.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..42ea6276b627a8738f2133e30a509dec2f25c28f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_abstract_algebra diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_anatomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45001d14cd72fefba40209a38ee044003b421b91 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_anatomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_astronomy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37183dc793c238e307ebf03cec66000336f50a45 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_astronomy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_business_ethics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbb5ea382e41273deec7c6df1840c3336c652a30 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_zh_other_tasks +task: global_mmlu_full_zh_business_ethics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_clinical_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d90ee0ea4fe299091b204e411d9a0ef1593fc0d2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_zh_other_tasks +task: global_mmlu_full_zh_clinical_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba2031feff45c3c7db26a8e30207cdff0bc81a1f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_college_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..860761b4f0079f95ca4646ad8c111d1542863a9e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_college_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53d01965f9d5cbbbeea2ccc9278809854b100d24 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_college_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbd2e4be875c004d729554a69a02b39671df8e50 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_college_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..523d6b3075fda6eb41abb160dad2c9a746fa9599 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_zh_other_tasks +task: global_mmlu_full_zh_college_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a08214f7fe96bedab8f3b1bb55fbb161b711119 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_college_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_computer_security.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99332b356f6e0fc4dc70ac228b6d8c7af51c3117 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_computer_security diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_conceptual_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b042cc8bf0231106ee6ecc283863ff8f577f2aab --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_conceptual_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_econometrics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf920112179d6d81df9c20512beb31a3422eaba4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_zh_social_sciences_tasks +task: global_mmlu_full_zh_econometrics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_electrical_engineering.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b30acad7621b2c8339eaa16c3bdb55fb6b9929a9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_electrical_engineering diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_elementary_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b108c421370e7f86d5aa9bb44a087f029e1aa6f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_elementary_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_formal_logic.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..647755991d84de30207ae9010f1a89f2ebc7d4a2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_zh_humanities_tasks +task: global_mmlu_full_zh_formal_logic diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_global_facts.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07d390aab2f0c30f68df1b6df744404902cb5d14 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_zh_other_tasks +task: global_mmlu_full_zh_global_facts diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_biology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28b2bdaa54cc1ab9dcad563983a4ab84abb25460 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_high_school_biology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_chemistry.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d084034da2abd8306485999ff67596a618c742f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_high_school_chemistry diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_computer_science.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6232ef607ecfcd8922518d269909d0a1a1b9dff4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_high_school_computer_science diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_european_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70e3e52bc9c834410672b8d2b534e35c4f4470b0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_zh_humanities_tasks +task: global_mmlu_full_zh_high_school_european_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_geography.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe6cb91391b8721579e1a9a1d0cfd4340d6fcd1c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_zh_social_sciences_tasks +task: global_mmlu_full_zh_high_school_geography diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_government_and_politics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfa7213a4b5e4f614c3cee431a3626875eb102a5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_zh_social_sciences_tasks +task: global_mmlu_full_zh_high_school_government_and_politics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_macroeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca0b7ad86f6facaca944eb360f7f5f6ed3d5c0f9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_zh_social_sciences_tasks +task: global_mmlu_full_zh_high_school_macroeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_mathematics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38868e969bf5d308b624dd6f4015605405aa9951 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_high_school_mathematics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_microeconomics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b79237d2f04a713f51bf1ec59bf5b6d7f0b5179d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_zh_social_sciences_tasks +task: global_mmlu_full_zh_high_school_microeconomics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_physics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6355da2f8e7196062ed7eda14ae58449daab18e7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_high_school_physics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f223886737a2157807ca44f74c717d6955d3f66a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_zh_social_sciences_tasks +task: global_mmlu_full_zh_high_school_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_statistics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9aac20977c68333b736f60f548ef4ca506f04960 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_high_school_statistics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_us_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47d8355f5bfe5d7905efd9e0b8f3c80babc88700 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_zh_humanities_tasks +task: global_mmlu_full_zh_high_school_us_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_world_history.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1f6671f2d183423812f1053d1e1052fc4a9aea0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_zh_humanities_tasks +task: global_mmlu_full_zh_high_school_world_history diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_human_aging.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6941ff7b2f110c7ea63fc623622f967802c3c8f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_zh_other_tasks +task: global_mmlu_full_zh_human_aging diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_human_sexuality.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee228b22d38c2c25a361f745ebdbafcbdfe58743 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_zh_social_sciences_tasks +task: global_mmlu_full_zh_human_sexuality diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_international_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07b1ebd16f4b5cfca32969742b2691c6ee3839f0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_zh_humanities_tasks +task: global_mmlu_full_zh_international_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_jurisprudence.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab10ffac5daa7950533669e07708f827099ee429 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_zh_humanities_tasks +task: global_mmlu_full_zh_jurisprudence diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_logical_fallacies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..451260b55f483511bd543a401881c29c3a9e6815 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_zh_humanities_tasks +task: global_mmlu_full_zh_logical_fallacies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_machine_learning.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..508d14f64d8f5fe198f36550d697f91100297c14 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_machine_learning diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_management.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9db0b32bb1f162c4d84b2a967465d1de9dd3fb17 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_zh_other_tasks +task: global_mmlu_full_zh_management diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_marketing.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7142ce49f5cb9f0deeae7e4b1d10a291c364220 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_zh_other_tasks +task: global_mmlu_full_zh_marketing diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_medical_genetics.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..220530907d9c9f952163216967fe7545dde8a072 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_zh_other_tasks +task: global_mmlu_full_zh_medical_genetics diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_miscellaneous.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b479c9b0cfd4160c92a4a71aae143789fc9c283 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_zh_other_tasks +task: global_mmlu_full_zh_miscellaneous diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_moral_disputes.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58d13a9944ad81bef3c761b55cede58c60571368 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_zh_humanities_tasks +task: global_mmlu_full_zh_moral_disputes diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_moral_scenarios.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95d91dfdebfd1f84481228d8fccee406399a43f1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_zh_humanities_tasks +task: global_mmlu_full_zh_moral_scenarios diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_nutrition.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57452a39b09498bc0f1c753dba424feb56bbef21 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_zh_other_tasks +task: global_mmlu_full_zh_nutrition diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_philosophy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20e237b24cdd4992e4c5e0c6ee1b191c5bc5a2b1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_zh_humanities_tasks +task: global_mmlu_full_zh_philosophy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_prehistory.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56358fe789834a394bea785e64dc49183cf080cb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_zh_humanities_tasks +task: global_mmlu_full_zh_prehistory diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_accounting.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..630681abf67acca1ce5b1bcc869f16229f5699ff --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_zh_other_tasks +task: global_mmlu_full_zh_professional_accounting diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_law.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e48f35cb38bad9514dad8bf685dc74dfc49951a2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_zh_humanities_tasks +task: global_mmlu_full_zh_professional_law diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_medicine.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f75432cd7d9c9d8c46050b669ac80f040e821df3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_zh_other_tasks +task: global_mmlu_full_zh_professional_medicine diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_psychology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fbbf45ad76940af904ca88b0ed6705ae5cf015b8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_zh_social_sciences_tasks +task: global_mmlu_full_zh_professional_psychology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_public_relations.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f760d2a29df46b1cfd1737f3304925477ed09a49 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_zh_social_sciences_tasks +task: global_mmlu_full_zh_public_relations diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_security_studies.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1dafaf5d754525a6ef06d5d3a6a61196a7c4982b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_zh_social_sciences_tasks +task: global_mmlu_full_zh_security_studies diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_sociology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..549f4ef156f784a95acd6a6d6bec93398944fb5d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_zh_social_sciences_tasks +task: global_mmlu_full_zh_sociology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_us_foreign_policy.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..597dcfa1cca601c17fce1facc62dcb3830220869 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_zh_social_sciences_tasks +task: global_mmlu_full_zh_us_foreign_policy diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_virology.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1984c6b00249308ba504b84a754025b38c422cac --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_zh_other_tasks +task: global_mmlu_full_zh_virology diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_world_religions.yaml b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa15c0cb282b9ff2bab168748b13a50b0de85d78 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/global_mmlu_full_zh_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_zh_humanities_tasks +task: global_mmlu_full_zh_world_religions diff --git a/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/utils.py b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_mmlu/full/zh/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/global_piqa/global_piqa_eng.yaml b/babylm_eval_repo/multilingual/tasks/global_piqa/global_piqa_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3754d6cbf293c412f9cb3afce753cdbb065208d6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_piqa/global_piqa_eng.yaml @@ -0,0 +1,10 @@ +group: global_piqa_eng +task: + - global_piqa_parallel_en + - global_piqa_nonparallel_en +aggregate_metric_list: + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_piqa/global_piqa_nld.yaml b/babylm_eval_repo/multilingual/tasks/global_piqa/global_piqa_nld.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f4fadd41100c61e00db8625e08ca5c4d4ed9fac --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_piqa/global_piqa_nld.yaml @@ -0,0 +1,10 @@ +group: global_piqa_nld +task: + - global_piqa_parallel_nl + - global_piqa_nonparallel_nl +aggregate_metric_list: + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_piqa/global_piqa_zho.yaml b/babylm_eval_repo/multilingual/tasks/global_piqa/global_piqa_zho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c50eacf140fc0d6b9b9c89d672df1eda753892 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_piqa/global_piqa_zho.yaml @@ -0,0 +1,10 @@ +group: global_piqa_zho +task: + - global_piqa_parallel_zh + - global_piqa_nonparallel_zh +aggregate_metric_list: + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_piqa_nonparallel/global_piqa_nonparallel_en.yaml b/babylm_eval_repo/multilingual/tasks/global_piqa_nonparallel/global_piqa_nonparallel_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24517c43a6951a4f47e60b6302e0886f45070268 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_piqa_nonparallel/global_piqa_nonparallel_en.yaml @@ -0,0 +1,20 @@ +task: global_piqa_nonparallel_en +dataset_path: mrlbenchmarks/global-piqa-nonparallel +dataset_name: eng_latn +output_type: multiple_choice +test_split: test +training_split: null +validation_split: null +doc_to_text: prompt +doc_to_target: label +doc_to_choice: '{{[solution0, solution1]}}' +# Token-count-normalized acc_norm (matches the strict sentence_zero_shot harness) instead of +# lm-eval's default character-length acc_norm. See tasks/global_piqa_nonparallel/utils.py. +process_results: !function utils.process_results +num_fewshot: 0 +metric_list: +- metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_piqa_nonparallel/global_piqa_nonparallel_nl.yaml b/babylm_eval_repo/multilingual/tasks/global_piqa_nonparallel/global_piqa_nonparallel_nl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8557737011a0e644235d6162e026dca24086e5c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_piqa_nonparallel/global_piqa_nonparallel_nl.yaml @@ -0,0 +1,3 @@ +include: global_piqa_nonparallel_en.yaml +task: global_piqa_nonparallel_nl +dataset_name: nld_latn diff --git a/babylm_eval_repo/multilingual/tasks/global_piqa_nonparallel/global_piqa_nonparallel_zh.yaml b/babylm_eval_repo/multilingual/tasks/global_piqa_nonparallel/global_piqa_nonparallel_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..469f11521646eb2ff9990f6f68f9a5205c527322 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_piqa_nonparallel/global_piqa_nonparallel_zh.yaml @@ -0,0 +1,3 @@ +include: global_piqa_nonparallel_en.yaml +task: global_piqa_nonparallel_zh +dataset_name: cmn_hans diff --git a/babylm_eval_repo/multilingual/tasks/global_piqa_nonparallel/utils.py b/babylm_eval_repo/multilingual/tasks/global_piqa_nonparallel/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d049ebfd3893b0a86074d9b91ba7cb0968a807f5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_piqa_nonparallel/utils.py @@ -0,0 +1,52 @@ +"""Custom token-count-normalized acc_norm for Global PIQA, matching the strict harness. + +lm-eval's built-in multiple_choice scorer normalizes by the *character* length of each choice +string (acc_norm) — see lm_eval/api/task.py. The strict sentence_zero_shot harness instead divides +the summed completion log-prob by the *token count* of the completion. To make the two harnesses +agree exactly, we override process_results and re-derive, per choice, the exact number of +continuation tokens lm-eval scored: + + n_tokens = len(encode(prompt + " " + choice)) - len(encode(prompt)) + +which equals the continuation slice lm-eval used (whole_enc[len(context_enc):]) and the strict +phrase-mask count. We count with add_special_tokens=False against the model's own tokenizer (located +via the GLOBAL_PIQA_TOKENIZER env var, set to the HF repo by scripts/global_piqa_model.sh): any +leading BOS / trailing EOS cancels in the subtraction, so the count matches lm-eval's continuation +slice whether or not the hf-bos BOS-fix is in use. The metric NAME stays `acc_norm` so the group +aggregation, collate_results.py and the markdown are unchanged; only its definition (token- vs +char-normalization) differs. +""" +import os +from functools import lru_cache + +import numpy as np +from transformers import AutoTokenizer + + +@lru_cache(maxsize=1) +def _tokenizer(): + path = os.environ.get("GLOBAL_PIQA_TOKENIZER") + if not path: + raise RuntimeError( + "GLOBAL_PIQA_TOKENIZER is not set; it must point at the model's tokenizer / HF repo " + "(set by scripts/global_piqa_model.sh)." + ) + return AutoTokenizer.from_pretrained(path, trust_remote_code=True) + + +def process_results(doc, results): + lls = np.array([r[0] for r in results]) # results are in doc_to_choice (solution0, solution1, ...) order + n = sum(1 for k in doc if k.startswith("solution")) + choices = [doc[f"solution{i}"] for i in range(n)] + + tok = _tokenizer() + prompt = doc["prompt"] + ctx_len = len(tok.encode(prompt, add_special_tokens=False)) # BOS/EOS cancel in the subtraction + tok_counts = np.array( + [max(len(tok.encode(prompt + " " + c, add_special_tokens=False)) - ctx_len, 1) for c in choices], + dtype=float, + ) + + pred_norm = int(np.argmax(lls / tok_counts)) + gold = doc["label"] + return {"acc_norm": 1.0 if pred_norm == gold else 0.0} diff --git a/babylm_eval_repo/multilingual/tasks/global_piqa_parallel/global_piqa_parallel_en.yaml b/babylm_eval_repo/multilingual/tasks/global_piqa_parallel/global_piqa_parallel_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5ce48c56ed93f1aa7e8f0ea385c3578243d92dd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_piqa_parallel/global_piqa_parallel_en.yaml @@ -0,0 +1,20 @@ +task: global_piqa_parallel_en +dataset_path: mrlbenchmarks/global-piqa-parallel +dataset_name: eng_latn +output_type: multiple_choice +test_split: test +training_split: null +validation_split: null +doc_to_text: prompt +doc_to_target: label +doc_to_choice: '{{[solution0, solution1, solution2, solution3]}}' +# Token-count-normalized acc_norm (matches the strict sentence_zero_shot harness) instead of +# lm-eval's default character-length acc_norm. See tasks/global_piqa_parallel/utils.py. +process_results: !function utils.process_results +num_fewshot: 0 +metric_list: +- metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/babylm_eval_repo/multilingual/tasks/global_piqa_parallel/global_piqa_parallel_nl.yaml b/babylm_eval_repo/multilingual/tasks/global_piqa_parallel/global_piqa_parallel_nl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..934845286fd48d20df065e8e75fbf52b6883c502 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_piqa_parallel/global_piqa_parallel_nl.yaml @@ -0,0 +1,3 @@ +include: global_piqa_parallel_en.yaml +task: global_piqa_parallel_nl +dataset_name: nld_latn diff --git a/babylm_eval_repo/multilingual/tasks/global_piqa_parallel/global_piqa_parallel_zh.yaml b/babylm_eval_repo/multilingual/tasks/global_piqa_parallel/global_piqa_parallel_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..520aac8735a0a124c5778d0243953c1e1d90c31e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_piqa_parallel/global_piqa_parallel_zh.yaml @@ -0,0 +1,3 @@ +include: global_piqa_parallel_en.yaml +task: global_piqa_parallel_zh +dataset_name: cmn_hans diff --git a/babylm_eval_repo/multilingual/tasks/global_piqa_parallel/utils.py b/babylm_eval_repo/multilingual/tasks/global_piqa_parallel/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d049ebfd3893b0a86074d9b91ba7cb0968a807f5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/global_piqa_parallel/utils.py @@ -0,0 +1,52 @@ +"""Custom token-count-normalized acc_norm for Global PIQA, matching the strict harness. + +lm-eval's built-in multiple_choice scorer normalizes by the *character* length of each choice +string (acc_norm) — see lm_eval/api/task.py. The strict sentence_zero_shot harness instead divides +the summed completion log-prob by the *token count* of the completion. To make the two harnesses +agree exactly, we override process_results and re-derive, per choice, the exact number of +continuation tokens lm-eval scored: + + n_tokens = len(encode(prompt + " " + choice)) - len(encode(prompt)) + +which equals the continuation slice lm-eval used (whole_enc[len(context_enc):]) and the strict +phrase-mask count. We count with add_special_tokens=False against the model's own tokenizer (located +via the GLOBAL_PIQA_TOKENIZER env var, set to the HF repo by scripts/global_piqa_model.sh): any +leading BOS / trailing EOS cancels in the subtraction, so the count matches lm-eval's continuation +slice whether or not the hf-bos BOS-fix is in use. The metric NAME stays `acc_norm` so the group +aggregation, collate_results.py and the markdown are unchanged; only its definition (token- vs +char-normalization) differs. +""" +import os +from functools import lru_cache + +import numpy as np +from transformers import AutoTokenizer + + +@lru_cache(maxsize=1) +def _tokenizer(): + path = os.environ.get("GLOBAL_PIQA_TOKENIZER") + if not path: + raise RuntimeError( + "GLOBAL_PIQA_TOKENIZER is not set; it must point at the model's tokenizer / HF repo " + "(set by scripts/global_piqa_model.sh)." + ) + return AutoTokenizer.from_pretrained(path, trust_remote_code=True) + + +def process_results(doc, results): + lls = np.array([r[0] for r in results]) # results are in doc_to_choice (solution0, solution1, ...) order + n = sum(1 for k in doc if k.startswith("solution")) + choices = [doc[f"solution{i}"] for i in range(n)] + + tok = _tokenizer() + prompt = doc["prompt"] + ctx_len = len(tok.encode(prompt, add_special_tokens=False)) # BOS/EOS cancel in the subtraction + tok_counts = np.array( + [max(len(tok.encode(prompt + " " + c, add_special_tokens=False)) - ctx_len, 1) for c in choices], + dtype=float, + ) + + pred_norm = int(np.argmax(lls / tok_counts)) + gold = doc["label"] + return {"acc_norm": 1.0 if pred_norm == gold else 0.0} diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/README.md b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0bebabc7d81fc41269fe3abfdaab8742ee9b9ac9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/README.md @@ -0,0 +1,108 @@ +# Gpqa + +### Paper + +Title: `MuBench: Assessment of Multilingual Capabilities of Large Language Models Across 61 Languages` + +Abstract: Multilingual large language models (LLMs) are advancing rapidly, with new models frequently claiming support for an increasing number of languages. However, existing evaluation datasets are limited and lack cross-lingual alignment, leaving assessments of multilingual capabilities fragmented in both language and skill coverage. To address this, we introduce MuBench, a benchmark covering 61 languages and evaluating a broad range of capabilities. We evaluate several state-of-the-art multilingual LLMs and find notable gaps between claimed and actual language coverage, particularly a persistent performance disparity between English and low-resource languages. Leveraging MuBench's alignment, we propose Multilingual Consistency (MLC) as a complementary metric to accuracy for analyzing performance bottlenecks and guiding model improvement. Finally, we pretrain a suite of 1.2B-parameter models on English and Chinese with 500B tokens, varying language ratios and parallel data proportions to investigate cross-lingual transfer dynamics. + +Homepage: https://huggingface.co/datasets/aialt/MuBench + + +### Citation + +``` +@misc{han2025mubench, + title={MuBench: Assessment of Multilingual Capabilities of Large Language Models Across 61 Languages}, + author={Wenhan Han and Yifan Zhang and Zhixun Chen and Binbin Liu and Haobin Lin and Bingni Zhang and Taifeng Wang and Mykola Pechenizkiy and Meng Fang and Yin Zheng}, + year={2025}, + eprint={2506.19468}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2506.19468}, +} +``` + +### Groups and Tasks + +#### Groups + +* `gpqa_mubench` + +#### Tasks +* `gpqa_af_mubench`: Afrikaans +* `gpqa_ar_mubench`: Arabic +* `gpqa_az_mubench`: Azerbaijani +* `gpqa_bg_mubench`: Bulgarian +* `gpqa_bn_mubench`: Bengali +* `gpqa_ca_mubench`: Catalan +* `gpqa_ceb_mubench`: Cebuano +* `gpqa_cs_mubench`: Czech +* `gpqa_da_mubench`: Danish +* `gpqa_de_mubench`: German +* `gpqa_el_mubench`: Greek +* `gpqa_en_mubench`: English +* `gpqa_es_mubench`: Spanish +* `gpqa_et_mubench`: Estonian +* `gpqa_fa_mubench`: Persian +* `gpqa_fi_mubench`: Finnish +* `gpqa_fr_mubench`: French +* `gpqa_ga_mubench`: Irish +* `gpqa_gu_mubench`: Gujarati +* `gpqa_he_mubench`: Hebrew +* `gpqa_hi_mubench`: Hindi +* `gpqa_hr_mubench`: Croatian +* `gpqa_hu_mubench`: Hungarian +* `gpqa_id_mubench`: Indonesian +* `gpqa_is_mubench`: Icelandic +* `gpqa_it_mubench`: Italian +* `gpqa_ja_mubench`: Japanese +* `gpqa_jv_mubench`: Javanese +* `gpqa_kk_mubench`: Kazakh +* `gpqa_km_mubench`: Khmer +* `gpqa_kn_mubench`: Kannada +* `gpqa_ko_mubench`: Korean +* `gpqa_lt_mubench`: Lithuanian +* `gpqa_lv_mubench`: Latvian +* `gpqa_ml_mubench`: Malayalam +* `gpqa_mr_mubench`: Marathi +* `gpqa_ms_mubench`: Malay +* `gpqa_my_mubench`: Myanmar +* `gpqa_nl_mubench`: Dutch +* `gpqa_no_mubench`: Norwegian +* `gpqa_pa_mubench`: Punjabi +* `gpqa_pl_mubench`: Polish +* `gpqa_pt_mubench`: Portuguese +* `gpqa_ro_mubench`: Romanian +* `gpqa_ru_mubench`: Russian +* `gpqa_sk_mubench`: Slovak +* `gpqa_sl_mubench`: Slovenian +* `gpqa_sq_mubench`: Albanian +* `gpqa_sr_mubench`: Serbian +* `gpqa_sv_mubench`: Swedish +* `gpqa_sw_mubench`: Swahili +* `gpqa_ta_mubench`: Tamil +* `gpqa_te_mubench`: Telugu +* `gpqa_th_mubench`: Thai +* `gpqa_tl_mubench`: Filipino +* `gpqa_tr_mubench`: Turkish +* `gpqa_uk_mubench`: Ukrainian +* `gpqa_ur_mubench`: Urdu +* `gpqa_uz_mubench`: Uzbek +* `gpqa_vi_mubench`: Vietnamese +* `gpqa_zh_mubench`: Chinese + + + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [ ] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/_gpqa_mubench.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/_gpqa_mubench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89650a2ad07d3e58ed2123b7a5a383dee3b5be87 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/_gpqa_mubench.yaml @@ -0,0 +1,72 @@ +group: gpqa_mubench +task: +- gpqa_af_mubench +- gpqa_ar_mubench +- gpqa_az_mubench +- gpqa_bg_mubench +- gpqa_bn_mubench +- gpqa_ca_mubench +- gpqa_ceb_mubench +- gpqa_cs_mubench +- gpqa_da_mubench +- gpqa_de_mubench +- gpqa_el_mubench +- gpqa_en_mubench +- gpqa_es_mubench +- gpqa_et_mubench +- gpqa_fa_mubench +- gpqa_fi_mubench +- gpqa_fr_mubench +- gpqa_ga_mubench +- gpqa_gu_mubench +- gpqa_he_mubench +- gpqa_hi_mubench +- gpqa_hr_mubench +- gpqa_hu_mubench +- gpqa_id_mubench +- gpqa_is_mubench +- gpqa_it_mubench +- gpqa_ja_mubench +- gpqa_jv_mubench +- gpqa_kk_mubench +- gpqa_km_mubench +- gpqa_kn_mubench +- gpqa_ko_mubench +- gpqa_lt_mubench +- gpqa_lv_mubench +- gpqa_ml_mubench +- gpqa_mr_mubench +- gpqa_ms_mubench +- gpqa_my_mubench +- gpqa_nl_mubench +- gpqa_no_mubench +- gpqa_pa_mubench +- gpqa_pl_mubench +- gpqa_pt_mubench +- gpqa_ro_mubench +- gpqa_ru_mubench +- gpqa_sk_mubench +- gpqa_sl_mubench +- gpqa_sq_mubench +- gpqa_sr_mubench +- gpqa_sv_mubench +- gpqa_sw_mubench +- gpqa_ta_mubench +- gpqa_te_mubench +- gpqa_th_mubench +- gpqa_tl_mubench +- gpqa_tr_mubench +- gpqa_uk_mubench +- gpqa_ur_mubench +- gpqa_uz_mubench +- gpqa_vi_mubench +- gpqa_zh_mubench +aggregate_metric_list: +- metric: acc + aggregation: mean + weight_by_size: true +- metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_af.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_af.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41b2186cb0798c098f39ef1b55500a3139ef2522 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_af.yaml @@ -0,0 +1,18 @@ +task: gpqa_af_mubench +dataset_path: aialt/MuBench +dataset_name: GPQADataset_lighteval_af +output_type: multiple_choice +validation_split: validation +test_split: test +doc_to_text: prompt +doc_to_target: label +doc_to_choice: choices +metric_list: +- metric: acc + aggregation: mean + higher_is_better: true +- metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ar.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79e78a3dbd55c7073bade46dea9440fa0f10a188 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ar.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_ar_mubench +dataset_name: GPQADataset_lighteval_ar diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_az.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_az.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66370ad21048e4b39febcd38169fa86d2b96f9c6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_az.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_az_mubench +dataset_name: GPQADataset_lighteval_az diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_bg.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_bg.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f5f4547a4e7f69fd7bdf1425096a5c58822b4c2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_bg.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_bg_mubench +dataset_name: GPQADataset_lighteval_bg diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_bn.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_bn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d2b97b4245c43f3a68159883b41bc28bec4eab81 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_bn.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_bn_mubench +dataset_name: GPQADataset_lighteval_bn diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ca.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e8a18706af5923a3dcab05092850f7b225cbbaf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ca.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_ca_mubench +dataset_name: GPQADataset_lighteval_ca diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ceb.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ceb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e69745563eb4526a7d3a67871e4608312744910c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ceb.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_ceb_mubench +dataset_name: GPQADataset_lighteval_ceb diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_cs.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_cs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88db0c04cd7ae31b97351b0c75cf0577e049498a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_cs.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_cs_mubench +dataset_name: GPQADataset_lighteval_cs diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_da.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_da.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8842e62bd4fb9796524be4e078b61ca28d56608 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_da.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_da_mubench +dataset_name: GPQADataset_lighteval_da diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_de.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a1797c5e7b2b435f9e8f97d76642a5b4d7af4c7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_de.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_de_mubench +dataset_name: GPQADataset_lighteval_de diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_el.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_el.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f5cd3204159c5cbcf796463b5c574c077ee6636 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_el.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_el_mubench +dataset_name: GPQADataset_lighteval_el diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_en.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee051a7957492aa7f23f13e3cdf1d530e956f411 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_en.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_en_mubench +dataset_name: GPQADataset_lighteval_en diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_es.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a918c7a8342ef3454b122d86a6f804d22a43bcdd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_es.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_es_mubench +dataset_name: GPQADataset_lighteval_es diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_et.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_et.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4f092c66aad690f588035a9dc00d32c26e7d84d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_et.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_et_mubench +dataset_name: GPQADataset_lighteval_et diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_fa.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_fa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d47b6c677d214aaaa1a4f3c7cc3361f0336f174c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_fa.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_fa_mubench +dataset_name: GPQADataset_lighteval_fa diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_fi.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_fi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da4fdadc186847f8b81d55f47f211d330038d004 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_fi.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_fi_mubench +dataset_name: GPQADataset_lighteval_fi diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_fr.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c39fcf37bce3e2080167ead36e18893bd827845 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_fr.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_fr_mubench +dataset_name: GPQADataset_lighteval_fr diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ga.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ga.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eaab96119e8be3b357f6720d730d8da4c38987f9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ga.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_ga_mubench +dataset_name: GPQADataset_lighteval_ga diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_gu.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_gu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..545b148dd0fa228e20f98a648dad0b5e37d75270 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_gu.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_gu_mubench +dataset_name: GPQADataset_lighteval_gu diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_he.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_he.yaml new file mode 100644 index 0000000000000000000000000000000000000000..446406d3dccc4157e81aa64e2e95d4dfec517b09 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_he.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_he_mubench +dataset_name: GPQADataset_lighteval_he diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_hi.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_hi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..729b96ae8926351e7d6481a14ce8b0b45f0a3dc6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_hi.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_hi_mubench +dataset_name: GPQADataset_lighteval_hi diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_hr.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_hr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8431c052953e50b634e7ad28ed9185a3c1abf5a9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_hr.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_hr_mubench +dataset_name: GPQADataset_lighteval_hr diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_hu.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_hu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1bd1c056fb2d44ba5208a07632b0fa5e9d6b1ea4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_hu.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_hu_mubench +dataset_name: GPQADataset_lighteval_hu diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_id.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_id.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d636927e42a75d60f35ccf1408b9b3e888a0f750 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_id.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_id_mubench +dataset_name: GPQADataset_lighteval_id diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_is.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_is.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91e17a8bdf5d1c40f014a7a8fd9ef9448ad23604 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_is.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_is_mubench +dataset_name: GPQADataset_lighteval_is diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_it.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_it.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3bdae1d47c8243d5b1cf6abb789b21240695975a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_it.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_it_mubench +dataset_name: GPQADataset_lighteval_it diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ja.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ja.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d0f3f3752ebdae349990127b9028fc3193640b2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ja.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_ja_mubench +dataset_name: GPQADataset_lighteval_ja diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_jv.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_jv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45faa6dc0a75e75c2ae7f6f7a38a1add408ceb81 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_jv.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_jv_mubench +dataset_name: GPQADataset_lighteval_jv diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_kk.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_kk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1c963e6da9db6a591fc7fe55ad89c1b9a925fd4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_kk.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_kk_mubench +dataset_name: GPQADataset_lighteval_kk diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_km.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_km.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a25332871def411670aab0285b3b6a69b08ffd3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_km.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_km_mubench +dataset_name: GPQADataset_lighteval_km diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_kn.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_kn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd9e008852e8331a6b0a9f79dd8985177a1a1355 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_kn.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_kn_mubench +dataset_name: GPQADataset_lighteval_kn diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ko.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ko.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6e66b531f3f1a7dc12bf9851269858435fa0bc4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ko.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_ko_mubench +dataset_name: GPQADataset_lighteval_ko diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_lt.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_lt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af7baf199a8b5644e2ecafc1556e509f000aca0d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_lt.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_lt_mubench +dataset_name: GPQADataset_lighteval_lt diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_lv.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_lv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0b405d077427c24a19bde713076dcea1e381230 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_lv.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_lv_mubench +dataset_name: GPQADataset_lighteval_lv diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ml.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ml.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef42fe28e9b6526ae8d003af12e9fb9f3e26d710 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ml.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_ml_mubench +dataset_name: GPQADataset_lighteval_ml diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_mr.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_mr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35b610c8c8eb5a8a0bb683d516e625ba86097222 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_mr.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_mr_mubench +dataset_name: GPQADataset_lighteval_mr diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ms.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ms.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa7c5b5cb98abc30d99962ca10a583450f41b596 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ms.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_ms_mubench +dataset_name: GPQADataset_lighteval_ms diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_my.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_my.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d65cb14d6b8c6d49e4677eb06d79291751ece5f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_my.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_my_mubench +dataset_name: GPQADataset_lighteval_my diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_nl.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_nl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..652325f5b98a83aea594006915c76709aff22009 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_nl.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_nl_mubench +dataset_name: GPQADataset_lighteval_nl diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_no.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_no.yaml new file mode 100644 index 0000000000000000000000000000000000000000..705494836809f285723e77c5323d5c946919f275 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_no.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_no_mubench +dataset_name: GPQADataset_lighteval_no diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_pa.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_pa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ccce1be260a778901cf394f8a6ff958cd8a4c8b5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_pa.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_pa_mubench +dataset_name: GPQADataset_lighteval_pa diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_pl.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_pl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88b0759120b5470e09b9048b21167bada2ba9b5c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_pl.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_pl_mubench +dataset_name: GPQADataset_lighteval_pl diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_pt.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5e04eca89faf34eeb760faebc8c9711362d7c1b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_pt.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_pt_mubench +dataset_name: GPQADataset_lighteval_pt diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ro.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ro.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b61d3acd12e9e8a98817ab72a4766574af67e71e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ro.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_ro_mubench +dataset_name: GPQADataset_lighteval_ro diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ru.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ru.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b266f131e2ea3865cb068cfbb9753e8716661664 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ru.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_ru_mubench +dataset_name: GPQADataset_lighteval_ru diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sk.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65f2f2954c1784edc32ad25b47402cd412942933 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sk.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_sk_mubench +dataset_name: GPQADataset_lighteval_sk diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sl.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c99123a277d3c22fc1b16ed62212d2e06883b2c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sl.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_sl_mubench +dataset_name: GPQADataset_lighteval_sl diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sq.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sq.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a49e10b594c66bbed50a9f6e15953006bdec13fa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sq.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_sq_mubench +dataset_name: GPQADataset_lighteval_sq diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sr.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d527a045e8aba77084f20c4c2265a0c271371c4d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sr.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_sr_mubench +dataset_name: GPQADataset_lighteval_sr diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sv.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76c88e274513ad43810a2606754fe93ec79b25b2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sv.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_sv_mubench +dataset_name: GPQADataset_lighteval_sv diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sw.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84660abc57c0b8bf40b5fe845c7105b5237418f1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_sw.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_sw_mubench +dataset_name: GPQADataset_lighteval_sw diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ta.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ta.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3371c60d44d09e42ebf0d081bb9b4be764824dc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ta.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_ta_mubench +dataset_name: GPQADataset_lighteval_ta diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_te.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_te.yaml new file mode 100644 index 0000000000000000000000000000000000000000..461a84ae61a016b375d746ee6233c490aa5b7b61 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_te.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_te_mubench +dataset_name: GPQADataset_lighteval_te diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_th.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_th.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b28cc1ce980cf5c536af18ccb5928a3f1b44860 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_th.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_th_mubench +dataset_name: GPQADataset_lighteval_th diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_tl.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_tl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bdbef30bbfe6d4723885ca38c561701e86b7d40 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_tl.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_tl_mubench +dataset_name: GPQADataset_lighteval_tl diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_tr.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_tr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d2d690d2915be9e138c23ce76719c02be69aa6a6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_tr.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_tr_mubench +dataset_name: GPQADataset_lighteval_tr diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_uk.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_uk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4c656b6d677d24bcd8cf42daa60d08234728e33 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_uk.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_uk_mubench +dataset_name: GPQADataset_lighteval_uk diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ur.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ur.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a4c531962eb9f6755057a8c3e5de518ad69694b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_ur.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_ur_mubench +dataset_name: GPQADataset_lighteval_ur diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_uz.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_uz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e5c2f9314c137222eed48e680793008284a7f30 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_uz.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_uz_mubench +dataset_name: GPQADataset_lighteval_uz diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_vi.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_vi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5094082f25d3b57e5a3dc04fca9e4b1e3df71fe6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_vi.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_vi_mubench +dataset_name: GPQADataset_lighteval_vi diff --git a/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_zh.yaml b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f26ea1b69bb92cfdfd46ff40c16033920901287 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/gpqa_mubench/default_zh.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: gpqa_zh_mubench +dataset_name: GPQADataset_lighteval_zh diff --git a/babylm_eval_repo/multilingual/tasks/hanzi/hanzi_pinyin.yaml b/babylm_eval_repo/multilingual/tasks/hanzi/hanzi_pinyin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88a51fe012dc2f7a72b59a5c4f58fe84c85d209f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hanzi/hanzi_pinyin.yaml @@ -0,0 +1,17 @@ +task: hanzi_pinyin +dataset_path: chinese-babylm-org/hanzi-pinyin +output_type: multiple_choice +training_split: null +validation_split: train +test_split: null +doc_to_text: '' +doc_to_target: 0 +doc_to_choice: '{{[sentence_good, sentence_bad]}}' +process_results: !function utils.process_results +num_fewshot: 0 +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/babylm_eval_repo/multilingual/tasks/hanzi/hanzi_structure.yaml b/babylm_eval_repo/multilingual/tasks/hanzi/hanzi_structure.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b4c4c225ee9dbf7f5a1ae341da4016d989e3077 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hanzi/hanzi_structure.yaml @@ -0,0 +1,17 @@ +task: hanzi_structure +dataset_path: chinese-babylm-org/hanzi-structure +output_type: multiple_choice +training_split: null +validation_split: train +test_split: null +doc_to_text: '' +doc_to_target: 0 +doc_to_choice: '{{[sentence_good, sentence_bad]}}' +process_results: !function utils.process_results +num_fewshot: 0 +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/babylm_eval_repo/multilingual/tasks/hanzi/utils.py b/babylm_eval_repo/multilingual/tasks/hanzi/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..dae00673414306068be715ef432da20045810c5f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hanzi/utils.py @@ -0,0 +1,38 @@ +"""Scoring helpers for the Chinese Hanzi minimal-pair tasks.""" + +import os +from functools import lru_cache + +import numpy as np +from transformers import AutoTokenizer + + +@lru_cache(maxsize=1) +def _tokenizer(): + model_name = os.environ.get("HANZI_TOKENIZER") + if not model_name: + raise RuntimeError( + "HANZI_TOKENIZER is not set. Run Hanzi tasks through " + "scripts/zeroshot_model.sh or set it to the model/tokenizer path." + ) + revision = os.environ.get("HANZI_TOKENIZER_REVISION", "main") + return AutoTokenizer.from_pretrained( + model_name, revision=revision, trust_remote_code=True + ) + + +def process_results(doc, results): + """Match the original Hanzi scorer, including its UNK-token policy.""" + tokenizer = _tokenizer() + unk_id = tokenizer.unk_token_id + if unk_id is not None and any( + unk_id in tokenizer.encode(sentence, add_special_tokens=False) + for sentence in (doc["sentence_good"], doc["sentence_bad"]) + ): + return {"acc": 0.0} + + log_likelihoods = np.array([result[0] for result in results]) + # gold is index 0; an exact tie (np.argmax would break it toward 0) counts + # as wrong. (UNK pairs are already returned as 0.0 above.) + top = np.flatnonzero(log_likelihoods == log_likelihoods.max()) + return {"acc": 1.0 if top.size == 1 and top[0] == 0 else 0.0} diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/README.md b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d1f7212678b9a78d512f0d29608a81628d4e0f47 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/README.md @@ -0,0 +1,108 @@ +# Hellaswag + +### Paper + +Title: `MuBench: Assessment of Multilingual Capabilities of Large Language Models Across 61 Languages` + +Abstract: Multilingual large language models (LLMs) are advancing rapidly, with new models frequently claiming support for an increasing number of languages. However, existing evaluation datasets are limited and lack cross-lingual alignment, leaving assessments of multilingual capabilities fragmented in both language and skill coverage. To address this, we introduce MuBench, a benchmark covering 61 languages and evaluating a broad range of capabilities. We evaluate several state-of-the-art multilingual LLMs and find notable gaps between claimed and actual language coverage, particularly a persistent performance disparity between English and low-resource languages. Leveraging MuBench's alignment, we propose Multilingual Consistency (MLC) as a complementary metric to accuracy for analyzing performance bottlenecks and guiding model improvement. Finally, we pretrain a suite of 1.2B-parameter models on English and Chinese with 500B tokens, varying language ratios and parallel data proportions to investigate cross-lingual transfer dynamics. + +Homepage: https://huggingface.co/datasets/aialt/MuBench + + +### Citation + +``` +@misc{han2025mubench, + title={MuBench: Assessment of Multilingual Capabilities of Large Language Models Across 61 Languages}, + author={Wenhan Han and Yifan Zhang and Zhixun Chen and Binbin Liu and Haobin Lin and Bingni Zhang and Taifeng Wang and Mykola Pechenizkiy and Meng Fang and Yin Zheng}, + year={2025}, + eprint={2506.19468}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2506.19468}, +} +``` + +### Groups and Tasks + +#### Groups + +* `hellaswag_mubench` + +#### Tasks +* `hellaswag_af_mubench`: Afrikaans +* `hellaswag_ar_mubench`: Arabic +* `hellaswag_az_mubench`: Azerbaijani +* `hellaswag_bg_mubench`: Bulgarian +* `hellaswag_bn_mubench`: Bengali +* `hellaswag_ca_mubench`: Catalan +* `hellaswag_ceb_mubench`: Cebuano +* `hellaswag_cs_mubench`: Czech +* `hellaswag_da_mubench`: Danish +* `hellaswag_de_mubench`: German +* `hellaswag_el_mubench`: Greek +* `hellaswag_en_mubench`: English +* `hellaswag_es_mubench`: Spanish +* `hellaswag_et_mubench`: Estonian +* `hellaswag_fa_mubench`: Persian +* `hellaswag_fi_mubench`: Finnish +* `hellaswag_fr_mubench`: French +* `hellaswag_ga_mubench`: Irish +* `hellaswag_gu_mubench`: Gujarati +* `hellaswag_he_mubench`: Hebrew +* `hellaswag_hi_mubench`: Hindi +* `hellaswag_hr_mubench`: Croatian +* `hellaswag_hu_mubench`: Hungarian +* `hellaswag_id_mubench`: Indonesian +* `hellaswag_is_mubench`: Icelandic +* `hellaswag_it_mubench`: Italian +* `hellaswag_ja_mubench`: Japanese +* `hellaswag_jv_mubench`: Javanese +* `hellaswag_kk_mubench`: Kazakh +* `hellaswag_km_mubench`: Khmer +* `hellaswag_kn_mubench`: Kannada +* `hellaswag_ko_mubench`: Korean +* `hellaswag_lt_mubench`: Lithuanian +* `hellaswag_lv_mubench`: Latvian +* `hellaswag_ml_mubench`: Malayalam +* `hellaswag_mr_mubench`: Marathi +* `hellaswag_ms_mubench`: Malay +* `hellaswag_my_mubench`: Myanmar +* `hellaswag_nl_mubench`: Dutch +* `hellaswag_no_mubench`: Norwegian +* `hellaswag_pa_mubench`: Punjabi +* `hellaswag_pl_mubench`: Polish +* `hellaswag_pt_mubench`: Portuguese +* `hellaswag_ro_mubench`: Romanian +* `hellaswag_ru_mubench`: Russian +* `hellaswag_sk_mubench`: Slovak +* `hellaswag_sl_mubench`: Slovenian +* `hellaswag_sq_mubench`: Albanian +* `hellaswag_sr_mubench`: Serbian +* `hellaswag_sv_mubench`: Swedish +* `hellaswag_sw_mubench`: Swahili +* `hellaswag_ta_mubench`: Tamil +* `hellaswag_te_mubench`: Telugu +* `hellaswag_th_mubench`: Thai +* `hellaswag_tl_mubench`: Filipino +* `hellaswag_tr_mubench`: Turkish +* `hellaswag_uk_mubench`: Ukrainian +* `hellaswag_ur_mubench`: Urdu +* `hellaswag_uz_mubench`: Uzbek +* `hellaswag_vi_mubench`: Vietnamese +* `hellaswag_zh_mubench`: Chinese + + + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [ ] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/_hellaswag_mubench.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/_hellaswag_mubench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62ccdcc158badc14d579384ade886f4b8332e379 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/_hellaswag_mubench.yaml @@ -0,0 +1,72 @@ +group: hellaswag_mubench +task: +- hellaswag_af_mubench +- hellaswag_ar_mubench +- hellaswag_az_mubench +- hellaswag_bg_mubench +- hellaswag_bn_mubench +- hellaswag_ca_mubench +- hellaswag_ceb_mubench +- hellaswag_cs_mubench +- hellaswag_da_mubench +- hellaswag_de_mubench +- hellaswag_el_mubench +- hellaswag_en_mubench +- hellaswag_es_mubench +- hellaswag_et_mubench +- hellaswag_fa_mubench +- hellaswag_fi_mubench +- hellaswag_fr_mubench +- hellaswag_ga_mubench +- hellaswag_gu_mubench +- hellaswag_he_mubench +- hellaswag_hi_mubench +- hellaswag_hr_mubench +- hellaswag_hu_mubench +- hellaswag_id_mubench +- hellaswag_is_mubench +- hellaswag_it_mubench +- hellaswag_ja_mubench +- hellaswag_jv_mubench +- hellaswag_kk_mubench +- hellaswag_km_mubench +- hellaswag_kn_mubench +- hellaswag_ko_mubench +- hellaswag_lt_mubench +- hellaswag_lv_mubench +- hellaswag_ml_mubench +- hellaswag_mr_mubench +- hellaswag_ms_mubench +- hellaswag_my_mubench +- hellaswag_nl_mubench +- hellaswag_no_mubench +- hellaswag_pa_mubench +- hellaswag_pl_mubench +- hellaswag_pt_mubench +- hellaswag_ro_mubench +- hellaswag_ru_mubench +- hellaswag_sk_mubench +- hellaswag_sl_mubench +- hellaswag_sq_mubench +- hellaswag_sr_mubench +- hellaswag_sv_mubench +- hellaswag_sw_mubench +- hellaswag_ta_mubench +- hellaswag_te_mubench +- hellaswag_th_mubench +- hellaswag_tl_mubench +- hellaswag_tr_mubench +- hellaswag_uk_mubench +- hellaswag_ur_mubench +- hellaswag_uz_mubench +- hellaswag_vi_mubench +- hellaswag_zh_mubench +aggregate_metric_list: +- metric: acc + aggregation: mean + weight_by_size: true +- metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_af.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_af.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac4e36402bd5d9b17ccbfbe63faa198446f49e50 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_af.yaml @@ -0,0 +1,19 @@ +task: hellaswag_af_mubench +dataset_path: aialt/MuBench +dataset_name: HellaswagDataset_lighteval_af +output_type: multiple_choice +validation_split: validation +test_split: test +doc_to_text: prompt +doc_to_target: label +doc_to_choice: choices +process_results: !function tie_handling.process_results_labeled +metric_list: +- metric: acc + aggregation: mean + higher_is_better: true +- metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ar.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..064747f5248f03afbdb89ec5c5c14493b209887a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ar.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_ar_mubench +dataset_name: HellaswagDataset_lighteval_ar diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_az.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_az.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4db72382d7a9ed75411c3264c67663b326d2f04 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_az.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_az_mubench +dataset_name: HellaswagDataset_lighteval_az diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_bg.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_bg.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe8483f792a3b75efd6062e20363cb4c5dcacb4e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_bg.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_bg_mubench +dataset_name: HellaswagDataset_lighteval_bg diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_bn.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_bn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a04dc560b01ac803a9402d0286c0c2c242a0657 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_bn.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_bn_mubench +dataset_name: HellaswagDataset_lighteval_bn diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ca.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f803550eca69c36ffef3f8421650b273be2db88 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ca.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_ca_mubench +dataset_name: HellaswagDataset_lighteval_ca diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ceb.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ceb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..489fdb3afba2062f9383e91e855145bcc9d4b1f9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ceb.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_ceb_mubench +dataset_name: HellaswagDataset_lighteval_ceb diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_cs.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_cs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93f14094d522d96cf79212892d26cac4276d09de --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_cs.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_cs_mubench +dataset_name: HellaswagDataset_lighteval_cs diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_da.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_da.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3b046923f6b021a9227bdebab744aa69165686a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_da.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_da_mubench +dataset_name: HellaswagDataset_lighteval_da diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_de.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5147c3311b560d869cd88c450d08d042990cf20d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_de.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_de_mubench +dataset_name: HellaswagDataset_lighteval_de diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_el.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_el.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c815b4352adcbf5359d750d14c223c63d5ef13c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_el.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_el_mubench +dataset_name: HellaswagDataset_lighteval_el diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_en.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62ca8c73d35927385c1b7ea3507f15cde5930f5c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_en.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_en_mubench +dataset_name: HellaswagDataset_lighteval_en diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_es.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..485e6fcfc935446049d2d660ae3f3abb4343ba6e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_es.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_es_mubench +dataset_name: HellaswagDataset_lighteval_es diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_et.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_et.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25b3c49b5be0f48f2ac4738ce8f323c2ac95bd79 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_et.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_et_mubench +dataset_name: HellaswagDataset_lighteval_et diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_fa.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_fa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..440cd787ace8bd0b22a0929223313a88d7d69b0f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_fa.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_fa_mubench +dataset_name: HellaswagDataset_lighteval_fa diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_fi.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_fi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ceea56263d9e4bcab8c5184c8f1be9de1098adb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_fi.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_fi_mubench +dataset_name: HellaswagDataset_lighteval_fi diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_fr.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2ed2e0f75e7fdffe3ddfa199e7063b6cc07bf76 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_fr.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_fr_mubench +dataset_name: HellaswagDataset_lighteval_fr diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ga.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ga.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae6874cf098da6ee03c9a1dccf0f4de0c9e33cb5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ga.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_ga_mubench +dataset_name: HellaswagDataset_lighteval_ga diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_gu.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_gu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be008b85ed6b5471ab98e61d0534dc1ea6725a7f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_gu.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_gu_mubench +dataset_name: HellaswagDataset_lighteval_gu diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_he.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_he.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d36d90fef98a65824aee15baff167f4eb247290 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_he.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_he_mubench +dataset_name: HellaswagDataset_lighteval_he diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_hi.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_hi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f7e019648655eec5e5b7e3e0b26cab74b546b93 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_hi.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_hi_mubench +dataset_name: HellaswagDataset_lighteval_hi diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_hr.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_hr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..593278e7b95c06aa88897aa20ee9f235a2d253f4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_hr.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_hr_mubench +dataset_name: HellaswagDataset_lighteval_hr diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_hu.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_hu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..611e9eb0872143295726b3249e0bb60f04f3cdd3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_hu.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_hu_mubench +dataset_name: HellaswagDataset_lighteval_hu diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_id.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_id.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4be44119d7489cdfd26a7e127e9e4d7dedb4ec9f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_id.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_id_mubench +dataset_name: HellaswagDataset_lighteval_id diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_is.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_is.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c5cce43a64f947a724ab472146ce0a74f16163f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_is.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_is_mubench +dataset_name: HellaswagDataset_lighteval_is diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_it.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_it.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80cad404b40bdcd2b23a7b2bca9b13fa037b9b77 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_it.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_it_mubench +dataset_name: HellaswagDataset_lighteval_it diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ja.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ja.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbded9d5eb5a8cb6d306a93c1bd54c790f240ae8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ja.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_ja_mubench +dataset_name: HellaswagDataset_lighteval_ja diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_jv.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_jv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae044f853b6ea73ded59bb6ad9a59add3e10abae --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_jv.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_jv_mubench +dataset_name: HellaswagDataset_lighteval_jv diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_kk.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_kk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..270e9533ae6570751d8e3f4a2c9c92a8fe881770 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_kk.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_kk_mubench +dataset_name: HellaswagDataset_lighteval_kk diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_km.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_km.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f879bd1bf064f47d87eb1a91298065472571d5d0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_km.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_km_mubench +dataset_name: HellaswagDataset_lighteval_km diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_kn.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_kn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bffa42796110cf0f62d375e770a38df0d5974485 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_kn.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_kn_mubench +dataset_name: HellaswagDataset_lighteval_kn diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ko.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ko.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc4dcc4da501925c34176879cc9b77353a470b6b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ko.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_ko_mubench +dataset_name: HellaswagDataset_lighteval_ko diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_lt.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_lt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ddc8e014fb08412a5ffff71270251a0c1e898bc7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_lt.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_lt_mubench +dataset_name: HellaswagDataset_lighteval_lt diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_lv.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_lv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0dc5b39d9e2f2d3d0418b1dfd35e8feec3926201 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_lv.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_lv_mubench +dataset_name: HellaswagDataset_lighteval_lv diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ml.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ml.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4e7de9436a25b44aadc29dda79113effb16a5a2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ml.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_ml_mubench +dataset_name: HellaswagDataset_lighteval_ml diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_mr.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_mr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f5217b29ec0a1980a136d36bfaa7e9800b56cf7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_mr.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_mr_mubench +dataset_name: HellaswagDataset_lighteval_mr diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ms.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ms.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a49cfd865942735e896ba0ea09fbecaa9db6dff5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ms.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_ms_mubench +dataset_name: HellaswagDataset_lighteval_ms diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_my.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_my.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ad26563eb3a08b9545cc69ba2ce11c61230dfdb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_my.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_my_mubench +dataset_name: HellaswagDataset_lighteval_my diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_nl.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_nl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79252751d0c03e118a16e363166e3ddc82bae5f5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_nl.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_nl_mubench +dataset_name: HellaswagDataset_lighteval_nl diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_no.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_no.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6109cd2f107b910c91b5568488be59fa6fdb7f2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_no.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_no_mubench +dataset_name: HellaswagDataset_lighteval_no diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_pa.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_pa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b237de931c429aa40daf74cadff0d2d53004fc9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_pa.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_pa_mubench +dataset_name: HellaswagDataset_lighteval_pa diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_pl.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_pl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b51817eafc42dea021672c9d3b8781f500f5a60 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_pl.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_pl_mubench +dataset_name: HellaswagDataset_lighteval_pl diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_pt.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4e898ff0eb9a455ccc8d6b9197bf16063a0e300 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_pt.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_pt_mubench +dataset_name: HellaswagDataset_lighteval_pt diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ro.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ro.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b958d9d71f09bfb19f8eaea494f54dc4299ab039 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ro.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_ro_mubench +dataset_name: HellaswagDataset_lighteval_ro diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ru.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ru.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b1b7914d7c84d8a6cfaad9c3c08b374e843c82f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ru.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_ru_mubench +dataset_name: HellaswagDataset_lighteval_ru diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sk.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9d8ec352c216eea803fe21890d8a1c9292fe26f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sk.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_sk_mubench +dataset_name: HellaswagDataset_lighteval_sk diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sl.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e4d55112cd807742caa50028df943f0bb1fba64 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sl.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_sl_mubench +dataset_name: HellaswagDataset_lighteval_sl diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sq.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sq.yaml new file mode 100644 index 0000000000000000000000000000000000000000..700f1d6d06faed24bba2284dce15555f549835dc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sq.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_sq_mubench +dataset_name: HellaswagDataset_lighteval_sq diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sr.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..968a69edb5e24c1a609e0c3eb5a9c00e74984cc3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sr.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_sr_mubench +dataset_name: HellaswagDataset_lighteval_sr diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sv.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2da1b7494f111cc640221ec6e35e128a7937a05 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sv.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_sv_mubench +dataset_name: HellaswagDataset_lighteval_sv diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sw.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd59f8482edcea4a634b254694067f14a83fefd9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_sw.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_sw_mubench +dataset_name: HellaswagDataset_lighteval_sw diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ta.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ta.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5877fd78c22707a3315fc753bc7f81d182f009b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ta.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_ta_mubench +dataset_name: HellaswagDataset_lighteval_ta diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_te.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_te.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c976a0b638ead6d4868a77ef033f019c2b27fbd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_te.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_te_mubench +dataset_name: HellaswagDataset_lighteval_te diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_th.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_th.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25999d3d8e6ff9e2f86ac83d61c5e96a037959a9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_th.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_th_mubench +dataset_name: HellaswagDataset_lighteval_th diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_tl.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_tl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3cdda4b355b1687ab9ee7f90cc369809fc684818 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_tl.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_tl_mubench +dataset_name: HellaswagDataset_lighteval_tl diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_tr.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_tr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d87e263f8002874db388ef63bc171a70cb4d1d2a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_tr.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_tr_mubench +dataset_name: HellaswagDataset_lighteval_tr diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_uk.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_uk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec4391ddb7d0e72f154452060e4db23d1502d3d5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_uk.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_uk_mubench +dataset_name: HellaswagDataset_lighteval_uk diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ur.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ur.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3962a7ca88d2a7d685988d0a1afa91120d21c7ed --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_ur.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_ur_mubench +dataset_name: HellaswagDataset_lighteval_ur diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_uz.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_uz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac854f7fb111d7587776c97d1953be217f08d5c2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_uz.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_uz_mubench +dataset_name: HellaswagDataset_lighteval_uz diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_vi.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_vi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3182e7236c54b3deba7398b73296e698a6b018f7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_vi.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_vi_mubench +dataset_name: HellaswagDataset_lighteval_vi diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_zh.yaml b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96a20f30d6195afd7a3c3c4192224c9d8b3b273e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/default_zh.yaml @@ -0,0 +1,3 @@ +include: default_af.yaml +task: hellaswag_zh_mubench +dataset_name: HellaswagDataset_lighteval_zh diff --git a/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/tie_handling.py b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/tie_handling.py new file mode 100644 index 0000000000000000000000000000000000000000..847dd843a3257ebd9e5359d9059d78255d48dad6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/hellaswag_mubench/tie_handling.py @@ -0,0 +1,50 @@ +"""Shared tie-aware scoring for the multilingual zeroshot log-prob tasks. + +This is the single source of truth. It is symlinked into each task directory as +``tie_handling.py`` because lm-eval's ``!function`` loads the module from the +yaml's *own* directory -- edit this file and every task picks it up. + +lm-eval's built-in multiple_choice ``acc`` uses ``np.argmax``, which breaks ties +toward the first choice. On tie-heavy models (e.g. UNK collapse) that silently +favours whichever answer sits at index 0. Here a k-way tie for the top score +gets ``1/k`` credit (0.5 for the usual 2-way tie): the fraction of an even split +over the tied choices that lands on gold -- the deterministic equivalent of a +coin flip on ties, with no RNG. + + process_results -- minimal-pair tasks (gold is index 0); acc only. + process_results_labeled -- labelled multiple choice (gold = doc["label"]); + acc and acc_norm (length-normalised). +""" + +import numpy as np + + +def _fractional(scores, gold): + top = np.flatnonzero(scores == scores.max()) + # k-way tie including gold => 1/k credit; gold not tied for top => wrong. + return (1.0 / top.size) if gold in top else 0.0 + + +def process_results(doc, results): + lls = np.array([r[0] for r in results], dtype=float) + return {"acc": _fractional(lls, 0)} + + +def _gold_index(doc, choices): + gold = doc["label"] + if isinstance(gold, str): + if gold in choices: + return choices.index(gold) + return int(gold) if gold.lstrip("-").isdigit() else -100 + return int(gold) + + +def process_results_labeled(doc, results): + lls = np.array([r[0] for r in results], dtype=float) + choices = doc["choices"] + gold = _gold_index(doc, choices) + completion_len = np.array([float(len(c)) for c in choices]) + return { + "acc": _fractional(lls, gold), + "acc_norm": _fractional(lls / completion_len, gold), + } diff --git a/babylm_eval_repo/multilingual/tasks/include/README.md b/babylm_eval_repo/multilingual/tasks/include/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1519bb5e32840118b4bec9259eb7eb77fc3f3a83 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/README.md @@ -0,0 +1,45 @@ +# INCLUDE + +### Paper + +Title: `INCLUDE: Evaluating Multilingual Language Understanding with Regional Knowledge` + +Abstract: [https://arxiv.org/abs/2411.19799](https://arxiv.org/abs/2411.19799) + +INCLUDE is a comprehensive knowledge- and reasoning-centric benchmark across 44 languages that evaluates multilingual LLMs for performance in the actual language environments where they would be deployed. It contains 22,637 4-option multiple-choice-questions (MCQ) extracted from academic and professional exams, covering 57 topics, including regional knowledge. + +> 🤗 [CohereLabs/include-base-44](https://huggingface.co/datasets/CohereLabs/include-base-44): Benchmark which supports 44 languages, each with 500 regional samples and 50 region-agnostic ones. + + +### Tasks: +We add the following evaluations: +- prompting with instructions in english for the 0-shot setting (`default`) +- prompting with instructions in english for the 5-shot setting (`few_shot_en`) +- prompting with instructions in the in-sample language for the 5-shot setting (`few_shot_og`) + + +### Languages + +Albanian, Arabic, Armenian, Azerbaijani, Basque, Belarusian, Bengali, Bulgarian, Chinese, Croatian, Dutch, Estonian, Finnish, French, Georgian, German, Greek, Hebrew, Hindi, Hungarian, Indonesia, Italian, Japanese, Kazakh, Korean, Lithuanian, Malay, Malayalam, Nepali, North Macedonian, Persian, Polish, Portuguese, russian, Serbian, Spanish, Tagalog, Tamil, Telugu, Turkish, Ukrainian, Urdu, Uzbek, Vietnamese + + +### Domains + +**Academic:** Accounting, Agriculture, Anthropology, Architecture and Design, Arts & Humanities, Biology, Business administration, Business ethics, Business, Chemistry, Computer Science, Culturology, Earth science, Economics, Education, Engineering, Environmental studies and forestry, Family and consumer science, Finance, Geography, Health, History, Human physical performance and recreation, Industrial and labor relations, International trade, Journalism, media studies, and communication, Language, Law, Library and museum studies, Literature, Logic, Management, Marketing, Math, Medicine, Military Sciences, Multiple exams, Performing arts, Philosophy, Physics, Political sciences, Psychology, Public Administration, Public Policy, Qualimetry, Religious studies, Risk management and insurance, Social Work, Social work, Sociology, STEM, Transportation, Visual Arts + +**Licenses:** Driving License, Marine License, Medical License, Professional Certifications + + +### Citation + +```bibtex + @article{romanou2024include, + title={INCLUDE: Evaluating Multilingual Language Understanding with Regional Knowledge}, + author={Angelika Romanou and Negar Foroutan and Anna Sotnikova and Zeming Chen and Sree Harsha Nelaturu and Shivalika Singh and Rishabh Maheshwary and Micol Altomare and Mohamed A Haggag and Imanol Schlag and Marzieh Fadaee and Sara Hooker and Antoine Bosselut and others}, + journal={ICLR}, + year={2024}, + primaryClass={cs.CL}, + eprint={2411.19799}, + url={https://arxiv.org/abs/2411.19799}, +} +``` diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Albanian/_albanian_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/_albanian_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..aca41ca7b302336d2274ba2ed7bc411c679b9576 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/_albanian_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Albanian +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Albanian/_include_base_44_albanian.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/_include_base_44_albanian.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5fa16f1fd2128dea7a0ab8775f0190b13b90ebc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/_include_base_44_albanian.yaml @@ -0,0 +1,12 @@ +group: include_base_44_albanian +task: +- include_base_44_albanian_arts_humanities +- include_base_44_albanian_stem +- include_base_44_albanian_business_commerce +- include_base_44_albanian_social_science +- include_base_44_albanian_health_oriented_education +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Albanian/include_base_44_albanian_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/include_base_44_albanian_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..875625a7354aacd9a9bc702b42f05f9dbd4ed9ed --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/include_base_44_albanian_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _albanian_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_albanian_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Albanian/include_base_44_albanian_business_commerce.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/include_base_44_albanian_business_commerce.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bddba42cf49b0ba4030e87de2f1ba67405478d0f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/include_base_44_albanian_business_commerce.yaml @@ -0,0 +1,4 @@ +include: _albanian_template_yaml +description: The following is multiple-choice question about Business & Commerce. +process_docs: !function 'utils.process_business_commerce' +task: include_base_44_albanian_business_commerce diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Albanian/include_base_44_albanian_health_oriented_education.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/include_base_44_albanian_health_oriented_education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..11ae70f5432d11be0a07a8377c679ab2c4d2683a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/include_base_44_albanian_health_oriented_education.yaml @@ -0,0 +1,4 @@ +include: _albanian_template_yaml +description: The following is multiple-choice question about Health oriented education. +process_docs: !function 'utils.process_health_oriented_education' +task: include_base_44_albanian_health_oriented_education diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Albanian/include_base_44_albanian_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/include_base_44_albanian_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f57e9e2f226e48a5c582be995344391de5de4273 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/include_base_44_albanian_social_science.yaml @@ -0,0 +1,4 @@ +include: _albanian_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_albanian_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Albanian/include_base_44_albanian_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/include_base_44_albanian_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01e4db1643edf4e4b306ffb631a2a69d9c6dcf54 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/include_base_44_albanian_stem.yaml @@ -0,0 +1,4 @@ +include: _albanian_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_albanian_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Albanian/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Albanian/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Arabic/_arabic_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/_arabic_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..4adf0456140d865e058f69e0d02249c1857af09a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/_arabic_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Arabic +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Arabic/_include_base_44_arabic.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/_include_base_44_arabic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ec8ce01f79640d5a3cc991994301de7d6bc3e62 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/_include_base_44_arabic.yaml @@ -0,0 +1,13 @@ +group: include_base_44_arabic +task: +- include_base_44_arabic_arts_humanities +- include_base_44_arabic_stem +- include_base_44_arabic_social_science +- include_base_44_arabic_driving_license +- include_base_44_arabic_general_knowledge +- include_base_44_arabic_business_commerce +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e2eddd208bced15cc1e5a5d28c7a28475228ef7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _arabic_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_arabic_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_business_commerce.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_business_commerce.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa360ffe0e1807a99d2bbfa50ffd253beab3785f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_business_commerce.yaml @@ -0,0 +1,4 @@ +include: _arabic_template_yaml +description: The following is multiple-choice question about Business & Commerce. +process_docs: !function 'utils.process_business_commerce' +task: include_base_44_arabic_business_commerce diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_driving_license.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_driving_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8139f83d377101703ef9c29915568ec65e18894f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_driving_license.yaml @@ -0,0 +1,4 @@ +include: _arabic_template_yaml +description: The following is multiple-choice question about Driving License. +process_docs: !function 'utils.process_driving_license' +task: include_base_44_arabic_driving_license diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_general_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_general_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc293ea412492aeddc85c86aa457abab6751b7ac --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_general_knowledge.yaml @@ -0,0 +1,4 @@ +include: _arabic_template_yaml +description: The following is multiple-choice question about General knowledge. +process_docs: !function 'utils.process_general_knowledge' +task: include_base_44_arabic_general_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a17da4505cb4118c73b169a5faca0fce2df905d9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_social_science.yaml @@ -0,0 +1,4 @@ +include: _arabic_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_arabic_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b3f4596f565c7fe35a2dc37191c8a2926068172 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/include_base_44_arabic_stem.yaml @@ -0,0 +1,4 @@ +include: _arabic_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_arabic_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Arabic/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Arabic/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Armenian/_armenian_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Armenian/_armenian_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..c37d7e4b1bc07f76c05ae50106aa377d60a413b1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Armenian/_armenian_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Armenian +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Armenian/_include_base_44_armenian.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Armenian/_include_base_44_armenian.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa2d2017d738343bbe51d5aa14683721a9225e1f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Armenian/_include_base_44_armenian.yaml @@ -0,0 +1,11 @@ +group: include_base_44_armenian +task: +- include_base_44_armenian_driving_license +- include_base_44_armenian_social_science +- include_base_44_armenian_arts_humanities +- include_base_44_armenian_stem +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Armenian/include_base_44_armenian_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Armenian/include_base_44_armenian_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa0db63822b00d1f54267982fef0450a48f32b30 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Armenian/include_base_44_armenian_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _armenian_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_armenian_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Armenian/include_base_44_armenian_driving_license.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Armenian/include_base_44_armenian_driving_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..250ad61ca275c6fb0d798cf2a750bb3e848a8028 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Armenian/include_base_44_armenian_driving_license.yaml @@ -0,0 +1,4 @@ +include: _armenian_template_yaml +description: The following is multiple-choice question about Driving License. +process_docs: !function 'utils.process_driving_license' +task: include_base_44_armenian_driving_license diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Armenian/include_base_44_armenian_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Armenian/include_base_44_armenian_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8249730e6b72cd8add6c9685666b60646b4cbd6d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Armenian/include_base_44_armenian_social_science.yaml @@ -0,0 +1,4 @@ +include: _armenian_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_armenian_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Armenian/include_base_44_armenian_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Armenian/include_base_44_armenian_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fec5401efe495bb081d1684ecf7c924fbd3af284 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Armenian/include_base_44_armenian_stem.yaml @@ -0,0 +1,4 @@ +include: _armenian_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_armenian_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Armenian/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Armenian/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Armenian/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/_azerbaijani_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/_azerbaijani_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..7bdf38a67666b2cb6a590fa5a8c4b3d4c2899a55 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/_azerbaijani_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Azerbaijani +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/_include_base_44_azerbaijani.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/_include_base_44_azerbaijani.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f28150235d100d6392e52c2e8b835e0c88b6016 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/_include_base_44_azerbaijani.yaml @@ -0,0 +1,13 @@ +group: include_base_44_azerbaijani +task: +- include_base_44_azerbaijani_business_commerce +- include_base_44_azerbaijani_applied_science +- include_base_44_azerbaijani_arts_humanities +- include_base_44_azerbaijani_social_science +- include_base_44_azerbaijani_stem +- include_base_44_azerbaijani_health_oriented_education +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_applied_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_applied_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3011c7a92aa0854494171cab09601b16e5d4290d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_applied_science.yaml @@ -0,0 +1,4 @@ +include: _azerbaijani_template_yaml +description: The following is multiple-choice question about Applied Science. +process_docs: !function 'utils.process_applied_science' +task: include_base_44_azerbaijani_applied_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a96a066005f741eb346404adb5ddd18ee789c7d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _azerbaijani_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_azerbaijani_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_business_commerce.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_business_commerce.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eebdc11d36eee1f02f8871b43c350b1a2e3cd3e0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_business_commerce.yaml @@ -0,0 +1,4 @@ +include: _azerbaijani_template_yaml +description: The following is multiple-choice question about Business & Commerce. +process_docs: !function 'utils.process_business_commerce' +task: include_base_44_azerbaijani_business_commerce diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_health_oriented_education.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_health_oriented_education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0944d70066d91607b672f52afd5b4a43039c8578 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_health_oriented_education.yaml @@ -0,0 +1,4 @@ +include: _azerbaijani_template_yaml +description: The following is multiple-choice question about Health oriented education. +process_docs: !function 'utils.process_health_oriented_education' +task: include_base_44_azerbaijani_health_oriented_education diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a619e1d89372860e64b474a2ad51f431b3d7790 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_social_science.yaml @@ -0,0 +1,4 @@ +include: _azerbaijani_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_azerbaijani_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b485c13dfe8076b33edf441152048208aab1d89 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/include_base_44_azerbaijani_stem.yaml @@ -0,0 +1,4 @@ +include: _azerbaijani_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_azerbaijani_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Azerbaijani/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Basque/_basque_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Basque/_basque_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f2b4a1e97fda0f3c9b3fdc5a5a871b6e753b04f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Basque/_basque_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Basque +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Basque/_include_base_44_basque.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Basque/_include_base_44_basque.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55e1dece2c8f8f2b7c27bdcc999757473aa5e607 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Basque/_include_base_44_basque.yaml @@ -0,0 +1,8 @@ +group: include_base_44_basque +task: +- include_base_44_basque_professional_certification +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Basque/include_base_44_basque_professional_certification.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Basque/include_base_44_basque_professional_certification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f02cd0f80dd28ba47462e9e613b085c83f980b7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Basque/include_base_44_basque_professional_certification.yaml @@ -0,0 +1,4 @@ +include: _basque_template_yaml +description: The following is multiple-choice question about Professional certification. +process_docs: !function 'utils.process_professional_certification' +task: include_base_44_basque_professional_certification diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Basque/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Basque/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Basque/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/_belarusian_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/_belarusian_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..29bf73f39d71a764c1c504826785e7d68d9207dc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/_belarusian_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Belarusian +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/_include_base_44_belarusian.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/_include_base_44_belarusian.yaml new file mode 100644 index 0000000000000000000000000000000000000000..916e0797134e20915ed0c109fc3337dff327b211 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/_include_base_44_belarusian.yaml @@ -0,0 +1,10 @@ +group: include_base_44_belarusian +task: +- include_base_44_belarusian_arts_humanities +- include_base_44_belarusian_social_science +- include_base_44_belarusian_stem +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/include_base_44_belarusian_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/include_base_44_belarusian_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d870b73925b422c079f81cb23c9e67ec9a9b7cb6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/include_base_44_belarusian_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _belarusian_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_belarusian_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/include_base_44_belarusian_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/include_base_44_belarusian_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90bda688d6fb10b8a5eee5e189b423975e800045 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/include_base_44_belarusian_social_science.yaml @@ -0,0 +1,4 @@ +include: _belarusian_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_belarusian_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/include_base_44_belarusian_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/include_base_44_belarusian_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d2748aa30f70b5b3edc75ad6b0c3b1a5a574917 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/include_base_44_belarusian_stem.yaml @@ -0,0 +1,4 @@ +include: _belarusian_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_belarusian_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Belarusian/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Bengali/_bengali_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Bengali/_bengali_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..3687066097bf2e69a7d043865ecfb79bca13e0f8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Bengali/_bengali_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Bengali +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Bengali/_include_base_44_bengali.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Bengali/_include_base_44_bengali.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a43fc3767e2bdd3c41ed95db93045cfb9798623 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Bengali/_include_base_44_bengali.yaml @@ -0,0 +1,11 @@ +group: include_base_44_bengali +task: +- include_base_44_bengali_arts_humanities +- include_base_44_bengali_stem +- include_base_44_bengali_general_knowledge +- include_base_44_bengali_professional_certification +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Bengali/include_base_44_bengali_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Bengali/include_base_44_bengali_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe8f72f9422098ca544dacf6cf48949db35b6ccb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Bengali/include_base_44_bengali_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _bengali_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_bengali_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Bengali/include_base_44_bengali_general_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Bengali/include_base_44_bengali_general_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0218f991fb3c3a2c6092c7612367ef3e63a59799 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Bengali/include_base_44_bengali_general_knowledge.yaml @@ -0,0 +1,4 @@ +include: _bengali_template_yaml +description: The following is multiple-choice question about General knowledge. +process_docs: !function 'utils.process_general_knowledge' +task: include_base_44_bengali_general_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Bengali/include_base_44_bengali_professional_certification.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Bengali/include_base_44_bengali_professional_certification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34017d88529a9fe297ec10ceca2c6c10280e8336 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Bengali/include_base_44_bengali_professional_certification.yaml @@ -0,0 +1,4 @@ +include: _bengali_template_yaml +description: The following is multiple-choice question about Professional certification. +process_docs: !function 'utils.process_professional_certification' +task: include_base_44_bengali_professional_certification diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Bengali/include_base_44_bengali_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Bengali/include_base_44_bengali_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e69db46de7bbf248eadcfceb2ac6a9da2c01a14e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Bengali/include_base_44_bengali_stem.yaml @@ -0,0 +1,4 @@ +include: _bengali_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_bengali_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Bengali/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Bengali/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Bengali/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/_bulgarian_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/_bulgarian_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e4159ee7e0d59f5019ffa780703a8bb737a2e24 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/_bulgarian_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Bulgarian +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/_include_base_44_bulgarian.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/_include_base_44_bulgarian.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a4d202a257eddcb50981cb8cadc837e3be0bf0e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/_include_base_44_bulgarian.yaml @@ -0,0 +1,10 @@ +group: include_base_44_bulgarian +task: +- include_base_44_bulgarian_stem +- include_base_44_bulgarian_arts_humanities +- include_base_44_bulgarian_social_science +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/include_base_44_bulgarian_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/include_base_44_bulgarian_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa37b61a3aaa16776ea229277eb3d926f1c67064 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/include_base_44_bulgarian_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _bulgarian_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_bulgarian_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/include_base_44_bulgarian_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/include_base_44_bulgarian_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d80dc62a2f36828e527c2b4a5e54cc59ca25f356 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/include_base_44_bulgarian_social_science.yaml @@ -0,0 +1,4 @@ +include: _bulgarian_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_bulgarian_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/include_base_44_bulgarian_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/include_base_44_bulgarian_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e730caf3384c0c3613316966a1cfde3c99cc6447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/include_base_44_bulgarian_stem.yaml @@ -0,0 +1,4 @@ +include: _bulgarian_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_bulgarian_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Bulgarian/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Chinese/_chinese_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/_chinese_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..c80d5f8009d82f160e947032025dc26309f60989 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/_chinese_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Chinese +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Chinese/_include_base_44_chinese.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/_include_base_44_chinese.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89d607b730d864d90afbb47b316d5451ecb6126e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/_include_base_44_chinese.yaml @@ -0,0 +1,15 @@ +group: include_base_44_chinese +task: +- include_base_44_chinese_applied_science +- include_base_44_chinese_health_oriented_education +- include_base_44_chinese_arts_humanities +- include_base_44_chinese_stem +- include_base_44_chinese_business_commerce +- include_base_44_chinese_professional_certification +- include_base_44_chinese_driving_license +- include_base_44_chinese_social_science +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_applied_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_applied_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1508e8dc7ea2e2f3509e3ef1bc2aaa8967705b9e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_applied_science.yaml @@ -0,0 +1,4 @@ +include: _chinese_template_yaml +description: The following is multiple-choice question about Applied Science. +process_docs: !function 'utils.process_applied_science' +task: include_base_44_chinese_applied_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f7365e90cf6b27f4259db0a43c71cb4f9e574df --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _chinese_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_chinese_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_business_commerce.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_business_commerce.yaml new file mode 100644 index 0000000000000000000000000000000000000000..354ef76e8a68e1570fe23a715e79077406cdd262 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_business_commerce.yaml @@ -0,0 +1,4 @@ +include: _chinese_template_yaml +description: The following is multiple-choice question about Business & Commerce. +process_docs: !function 'utils.process_business_commerce' +task: include_base_44_chinese_business_commerce diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_driving_license.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_driving_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e58f0ae64d7216bd0929ecdd50c872a64a6958c0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_driving_license.yaml @@ -0,0 +1,4 @@ +include: _chinese_template_yaml +description: The following is multiple-choice question about Driving License. +process_docs: !function 'utils.process_driving_license' +task: include_base_44_chinese_driving_license diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_health_oriented_education.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_health_oriented_education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..639ee5427ac247f6b0633809b650d827d577096d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_health_oriented_education.yaml @@ -0,0 +1,4 @@ +include: _chinese_template_yaml +description: The following is multiple-choice question about Health oriented education. +process_docs: !function 'utils.process_health_oriented_education' +task: include_base_44_chinese_health_oriented_education diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_professional_certification.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_professional_certification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e537e09dcfc4f5b94cb2a9f71d6fcbbd01002217 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_professional_certification.yaml @@ -0,0 +1,4 @@ +include: _chinese_template_yaml +description: The following is multiple-choice question about Professional certification. +process_docs: !function 'utils.process_professional_certification' +task: include_base_44_chinese_professional_certification diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..00f45e91a0b18ad74b1440caa259009bb8f5e59f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_social_science.yaml @@ -0,0 +1,4 @@ +include: _chinese_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_chinese_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17627f2ec80270440e1bf104f14f1caef3b0293e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/include_base_44_chinese_stem.yaml @@ -0,0 +1,4 @@ +include: _chinese_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_chinese_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Chinese/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Chinese/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Croatian/_croatian_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Croatian/_croatian_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e2ddf960949e7c913eff2887ce7f2ac39cd1f7e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Croatian/_croatian_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Croatian +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Croatian/_include_base_44_croatian.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Croatian/_include_base_44_croatian.yaml new file mode 100644 index 0000000000000000000000000000000000000000..157850d78ec3a753f5c25aabd4b67bd8a25e4763 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Croatian/_include_base_44_croatian.yaml @@ -0,0 +1,10 @@ +group: include_base_44_croatian +task: +- include_base_44_croatian_stem +- include_base_44_croatian_arts_humanities +- include_base_44_croatian_social_science +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Croatian/include_base_44_croatian_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Croatian/include_base_44_croatian_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c5c4a2a59d949db66974514af5da2e88a0fdc99 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Croatian/include_base_44_croatian_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _croatian_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_croatian_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Croatian/include_base_44_croatian_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Croatian/include_base_44_croatian_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e81897cb25ff7b7ffd910cec2b7cfe8e5cc3467 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Croatian/include_base_44_croatian_social_science.yaml @@ -0,0 +1,4 @@ +include: _croatian_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_croatian_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Croatian/include_base_44_croatian_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Croatian/include_base_44_croatian_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a3693647e25a41b5b0184f5a07950e378736aa9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Croatian/include_base_44_croatian_stem.yaml @@ -0,0 +1,4 @@ +include: _croatian_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_croatian_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Croatian/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Croatian/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Croatian/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Dutch/_dutch_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/_dutch_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..93481ed28d1bcd7d84dc66eb79c1caedc87550aa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/_dutch_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Dutch +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Dutch/_include_base_44_dutch.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/_include_base_44_dutch.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c01351f099f8e3ba4ade0806a4363113625fae6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/_include_base_44_dutch.yaml @@ -0,0 +1,12 @@ +group: include_base_44_dutch +task: +- include_base_44_dutch_arts_humanities +- include_base_44_dutch_stem +- include_base_44_dutch_social_science +- include_base_44_dutch_health_oriented_education +- include_base_44_dutch_applied_science +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Dutch/include_base_44_dutch_applied_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/include_base_44_dutch_applied_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0289fc60e2e01fa23b7b9333e3bab434be1cc842 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/include_base_44_dutch_applied_science.yaml @@ -0,0 +1,4 @@ +include: _dutch_template_yaml +description: The following is multiple-choice question about Applied Science. +process_docs: !function 'utils.process_applied_science' +task: include_base_44_dutch_applied_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Dutch/include_base_44_dutch_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/include_base_44_dutch_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30013d095eabf5b77c70d97e5c12c52b90a4cbb6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/include_base_44_dutch_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _dutch_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_dutch_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Dutch/include_base_44_dutch_health_oriented_education.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/include_base_44_dutch_health_oriented_education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..deebc60dde9eb8ce89624b468f54fdd3c364feb8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/include_base_44_dutch_health_oriented_education.yaml @@ -0,0 +1,4 @@ +include: _dutch_template_yaml +description: The following is multiple-choice question about Health oriented education. +process_docs: !function 'utils.process_health_oriented_education' +task: include_base_44_dutch_health_oriented_education diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Dutch/include_base_44_dutch_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/include_base_44_dutch_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c1635229dcd54d0e0ea8d98271f773614e11887 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/include_base_44_dutch_social_science.yaml @@ -0,0 +1,4 @@ +include: _dutch_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_dutch_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Dutch/include_base_44_dutch_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/include_base_44_dutch_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08ed4b543ce50559198152dea42cf3e861c1afa2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/include_base_44_dutch_stem.yaml @@ -0,0 +1,4 @@ +include: _dutch_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_dutch_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Dutch/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Dutch/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Estonian/_estonian_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/_estonian_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..45e7f97f255836718c235d95fa28b1bceedcce3d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/_estonian_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Estonian +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Estonian/_include_base_44_estonian.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/_include_base_44_estonian.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82a0a39aa1cbca53839708943b852a4eeffcc4aa --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/_include_base_44_estonian.yaml @@ -0,0 +1,12 @@ +group: include_base_44_estonian +task: +- include_base_44_estonian_health_oriented_education +- include_base_44_estonian_social_science +- include_base_44_estonian_applied_science +- include_base_44_estonian_stem +- include_base_44_estonian_arts_humanities +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Estonian/include_base_44_estonian_applied_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/include_base_44_estonian_applied_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f88eedc1ba73ae7643cfc6dbb9305c6d5994e559 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/include_base_44_estonian_applied_science.yaml @@ -0,0 +1,4 @@ +include: _estonian_template_yaml +description: The following is multiple-choice question about Applied Science. +process_docs: !function 'utils.process_applied_science' +task: include_base_44_estonian_applied_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Estonian/include_base_44_estonian_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/include_base_44_estonian_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f95bdfbfde40e5e2fd508da6a5470188ae11ec62 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/include_base_44_estonian_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _estonian_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_estonian_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Estonian/include_base_44_estonian_health_oriented_education.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/include_base_44_estonian_health_oriented_education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9915b94131dda1fb94e4e0e4afb857c971e3fb16 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/include_base_44_estonian_health_oriented_education.yaml @@ -0,0 +1,4 @@ +include: _estonian_template_yaml +description: The following is multiple-choice question about Health oriented education. +process_docs: !function 'utils.process_health_oriented_education' +task: include_base_44_estonian_health_oriented_education diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Estonian/include_base_44_estonian_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/include_base_44_estonian_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba29726c19ac61605ec4b1f467bae88b8b160754 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/include_base_44_estonian_social_science.yaml @@ -0,0 +1,4 @@ +include: _estonian_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_estonian_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Estonian/include_base_44_estonian_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/include_base_44_estonian_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcf80871829f60b8dd9283259c170215b4c4574c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/include_base_44_estonian_stem.yaml @@ -0,0 +1,4 @@ +include: _estonian_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_estonian_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Estonian/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Estonian/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Finnish/_finnish_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/_finnish_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d5501329920561986dabac6d654fc455d3f24b5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/_finnish_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Finnish +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Finnish/_include_base_44_finnish.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/_include_base_44_finnish.yaml new file mode 100644 index 0000000000000000000000000000000000000000..831c8d9a73768479f7921f2fca921aff3e2aed73 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/_include_base_44_finnish.yaml @@ -0,0 +1,12 @@ +group: include_base_44_finnish +task: +- include_base_44_finnish_stem +- include_base_44_finnish_arts_humanities +- include_base_44_finnish_social_science +- include_base_44_finnish_health_oriented_education +- include_base_44_finnish_applied_science +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Finnish/include_base_44_finnish_applied_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/include_base_44_finnish_applied_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50ec3d3416aa50b467134bc0db0660373acb16e6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/include_base_44_finnish_applied_science.yaml @@ -0,0 +1,4 @@ +include: _finnish_template_yaml +description: The following is multiple-choice question about Applied Science. +process_docs: !function 'utils.process_applied_science' +task: include_base_44_finnish_applied_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Finnish/include_base_44_finnish_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/include_base_44_finnish_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5e2815d68b75711937c7b57c7455335fb68035f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/include_base_44_finnish_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _finnish_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_finnish_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Finnish/include_base_44_finnish_health_oriented_education.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/include_base_44_finnish_health_oriented_education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf8d9b7e91932163a5931410a557f8e6f2e81a86 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/include_base_44_finnish_health_oriented_education.yaml @@ -0,0 +1,4 @@ +include: _finnish_template_yaml +description: The following is multiple-choice question about Health oriented education. +process_docs: !function 'utils.process_health_oriented_education' +task: include_base_44_finnish_health_oriented_education diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Finnish/include_base_44_finnish_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/include_base_44_finnish_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41f447b6b97c46f812f0221dde214a2d173fbbb5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/include_base_44_finnish_social_science.yaml @@ -0,0 +1,4 @@ +include: _finnish_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_finnish_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Finnish/include_base_44_finnish_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/include_base_44_finnish_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5056056245dd2d8d84b70248e18bf86a458c6ef3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/include_base_44_finnish_stem.yaml @@ -0,0 +1,4 @@ +include: _finnish_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_finnish_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Finnish/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Finnish/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/French/_french_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/French/_french_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..b68826bd9df4e1e1a4059fc0a6f48224a15d1d8a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/French/_french_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: French +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/French/_include_base_44_french.yaml b/babylm_eval_repo/multilingual/tasks/include/default/French/_include_base_44_french.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40012728edbc6e4ce931331eb630adcde7ac6bda --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/French/_include_base_44_french.yaml @@ -0,0 +1,12 @@ +group: include_base_44_french +task: +- include_base_44_french_stem +- include_base_44_french_social_science +- include_base_44_french_health_oriented_education +- include_base_44_french_arts_humanities +- include_base_44_french_driving_license +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/French/include_base_44_french_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/French/include_base_44_french_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62153e34984329b52a7d96253d57ab909463e221 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/French/include_base_44_french_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _french_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_french_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/French/include_base_44_french_driving_license.yaml b/babylm_eval_repo/multilingual/tasks/include/default/French/include_base_44_french_driving_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a8b9b12ac125631b24baddd8988db0ade36b5bf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/French/include_base_44_french_driving_license.yaml @@ -0,0 +1,4 @@ +include: _french_template_yaml +description: The following is multiple-choice question about Driving License. +process_docs: !function 'utils.process_driving_license' +task: include_base_44_french_driving_license diff --git a/babylm_eval_repo/multilingual/tasks/include/default/French/include_base_44_french_health_oriented_education.yaml b/babylm_eval_repo/multilingual/tasks/include/default/French/include_base_44_french_health_oriented_education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be22467a701d509aa4a57635f7f1f3f41b1b11f8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/French/include_base_44_french_health_oriented_education.yaml @@ -0,0 +1,4 @@ +include: _french_template_yaml +description: The following is multiple-choice question about Health oriented education. +process_docs: !function 'utils.process_health_oriented_education' +task: include_base_44_french_health_oriented_education diff --git a/babylm_eval_repo/multilingual/tasks/include/default/French/include_base_44_french_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/French/include_base_44_french_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1efa37dfdd58ab221f7a05f903adb9b787f9eae --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/French/include_base_44_french_social_science.yaml @@ -0,0 +1,4 @@ +include: _french_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_french_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/French/include_base_44_french_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/French/include_base_44_french_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aee5db150e4c7fb7094f9be727c23e5991574eae --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/French/include_base_44_french_stem.yaml @@ -0,0 +1,4 @@ +include: _french_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_french_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/French/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/French/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/French/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Georgian/_georgian_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Georgian/_georgian_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..35b6b81a8828c11491e734e265eef35743c50caf --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Georgian/_georgian_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Georgian +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Georgian/_include_base_44_georgian.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Georgian/_include_base_44_georgian.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16b1306ff6f727a48a4aa9c2ef1c175747ea54f4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Georgian/_include_base_44_georgian.yaml @@ -0,0 +1,8 @@ +group: include_base_44_georgian +task: +- include_base_44_georgian_arts_humanities +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Georgian/include_base_44_georgian_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Georgian/include_base_44_georgian_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0b049fc7bec5af18e8fb9ff52870cd60a901fa1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Georgian/include_base_44_georgian_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _georgian_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_georgian_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Georgian/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Georgian/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Georgian/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/German/_german_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/German/_german_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..c35c665000db0282ff98cfa3313a836aee907ee9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/German/_german_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: German +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/German/_include_base_44_german.yaml b/babylm_eval_repo/multilingual/tasks/include/default/German/_include_base_44_german.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c06efcb6422cb53f16b0e5df069e5b2085164909 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/German/_include_base_44_german.yaml @@ -0,0 +1,10 @@ +group: include_base_44_german +task: +- include_base_44_german_stem +- include_base_44_german_social_science +- include_base_44_german_driving_license +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/German/include_base_44_german_driving_license.yaml b/babylm_eval_repo/multilingual/tasks/include/default/German/include_base_44_german_driving_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3ac238dd28f7a261c1f7451106849dc48d8f2b6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/German/include_base_44_german_driving_license.yaml @@ -0,0 +1,4 @@ +include: _german_template_yaml +description: The following is multiple-choice question about Driving License. +process_docs: !function 'utils.process_driving_license' +task: include_base_44_german_driving_license diff --git a/babylm_eval_repo/multilingual/tasks/include/default/German/include_base_44_german_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/German/include_base_44_german_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8faa76339ae160c971f856c22a6f87720acfd7d5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/German/include_base_44_german_social_science.yaml @@ -0,0 +1,4 @@ +include: _german_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_german_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/German/include_base_44_german_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/German/include_base_44_german_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06ed2e7ba42c24d572d72a06fba1422650cc62d8 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/German/include_base_44_german_stem.yaml @@ -0,0 +1,4 @@ +include: _german_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_german_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/German/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/German/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/German/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Greek/_greek_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Greek/_greek_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ee70e8539ed9e97e9e03b18a4784c4052325b48 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Greek/_greek_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Greek +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Greek/_include_base_44_greek.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Greek/_include_base_44_greek.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77772f269eb3ad9f79e12a84e62c6fe201094f9d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Greek/_include_base_44_greek.yaml @@ -0,0 +1,14 @@ +group: include_base_44_greek +task: +- include_base_44_greek_stem +- include_base_44_greek_arts_humanities +- include_base_44_greek_social_science +- include_base_44_greek_business_commerce +- include_base_44_greek_health_oriented_education +- include_base_44_greek_professional_certification +- include_base_44_greek_medical_license +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0f58aa0d3e6957c1e19f8a751aedbf22cd4a9d7 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _greek_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_greek_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_business_commerce.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_business_commerce.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f10daa10a0316454a3c890de8de0702c904d06dd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_business_commerce.yaml @@ -0,0 +1,4 @@ +include: _greek_template_yaml +description: The following is multiple-choice question about Business & Commerce. +process_docs: !function 'utils.process_business_commerce' +task: include_base_44_greek_business_commerce diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_health_oriented_education.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_health_oriented_education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a0d0e15d9f0ec4e7ceb40ae6f515104e078df78 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_health_oriented_education.yaml @@ -0,0 +1,4 @@ +include: _greek_template_yaml +description: The following is multiple-choice question about Health oriented education. +process_docs: !function 'utils.process_health_oriented_education' +task: include_base_44_greek_health_oriented_education diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_medical_license.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_medical_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7607dc9e7e874476077cbf9882963973e0185f3d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_medical_license.yaml @@ -0,0 +1,4 @@ +include: _greek_template_yaml +description: The following is multiple-choice question about Medical License. +process_docs: !function 'utils.process_medical_license' +task: include_base_44_greek_medical_license diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_professional_certification.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_professional_certification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b62bf01fec370e8737c9d6a63209283810914bb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_professional_certification.yaml @@ -0,0 +1,4 @@ +include: _greek_template_yaml +description: The following is multiple-choice question about Professional certification. +process_docs: !function 'utils.process_professional_certification' +task: include_base_44_greek_professional_certification diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..efe82841059c3b944f43e31f5a762c88163d3d2c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_social_science.yaml @@ -0,0 +1,4 @@ +include: _greek_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_greek_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5e56e27e0afa8ddfab9477baae493956ba644d6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Greek/include_base_44_greek_stem.yaml @@ -0,0 +1,4 @@ +include: _greek_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_greek_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Greek/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Greek/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Greek/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hebrew/_hebrew_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hebrew/_hebrew_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..703b7f1dbcecc3e8e2e321b23b4a11bb520e7561 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hebrew/_hebrew_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Hebrew +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hebrew/_include_base_44_hebrew.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hebrew/_include_base_44_hebrew.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36f44731c22b6800d7a9174712f5470b8734626e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hebrew/_include_base_44_hebrew.yaml @@ -0,0 +1,9 @@ +group: include_base_44_hebrew +task: +- include_base_44_hebrew_arts_humanities +- include_base_44_hebrew_driving_license +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hebrew/include_base_44_hebrew_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hebrew/include_base_44_hebrew_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1bbe72365b248c52ab933f1467bb2fb1a992655 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hebrew/include_base_44_hebrew_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _hebrew_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_hebrew_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hebrew/include_base_44_hebrew_driving_license.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hebrew/include_base_44_hebrew_driving_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3308e8ab4ad92a1f5e84720e479fcba4e938ee03 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hebrew/include_base_44_hebrew_driving_license.yaml @@ -0,0 +1,4 @@ +include: _hebrew_template_yaml +description: The following is multiple-choice question about Driving license. +process_docs: !function 'utils.process_driving_license' +task: include_base_44_hebrew_driving_license diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hebrew/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Hebrew/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hebrew/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hindi/_hindi_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/_hindi_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0b343399fe81150db0f11ff7415e7f18269d206 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/_hindi_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Hindi +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hindi/_include_base_44_hindi.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/_include_base_44_hindi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..603f14c982b0032bf0865204905baf95a08409a0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/_include_base_44_hindi.yaml @@ -0,0 +1,15 @@ +group: include_base_44_hindi +task: +- include_base_44_hindi_professional_certification +- include_base_44_hindi_stem +- include_base_44_hindi_social_science +- include_base_44_hindi_driving_license +- include_base_44_hindi_applied_science +- include_base_44_hindi_arts_humanities +- include_base_44_hindi_general_knowledge +- include_base_44_hindi_health_oriented_education +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_applied_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_applied_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e79ca38b2ca350b186d9f68b370fd3049fc1a2ae --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_applied_science.yaml @@ -0,0 +1,4 @@ +include: _hindi_template_yaml +description: The following is multiple-choice question about Applied Science. +process_docs: !function 'utils.process_applied_science' +task: include_base_44_hindi_applied_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19fb26c7d40cbd263800e06f402506eaecab02d0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _hindi_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_hindi_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_driving_license.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_driving_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5518ec4df43923b97366d7bb48afe68e0d46e088 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_driving_license.yaml @@ -0,0 +1,4 @@ +include: _hindi_template_yaml +description: The following is multiple-choice question about Driving License. +process_docs: !function 'utils.process_driving_license' +task: include_base_44_hindi_driving_license diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_general_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_general_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f1af9151b0d2d874369a6034b75987f67ae9d1c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_general_knowledge.yaml @@ -0,0 +1,4 @@ +include: _hindi_template_yaml +description: The following is multiple-choice question about General knowledge. +process_docs: !function 'utils.process_general_knowledge' +task: include_base_44_hindi_general_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_health_oriented_education.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_health_oriented_education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0faad17dd25d346f950a306c77e9f4354d7cd9a0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_health_oriented_education.yaml @@ -0,0 +1,4 @@ +include: _hindi_template_yaml +description: The following is multiple-choice question about Health oriented education. +process_docs: !function 'utils.process_health_oriented_education' +task: include_base_44_hindi_health_oriented_education diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_professional_certification.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_professional_certification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0610a13394a535309020cc9a60cd2fea5cb4b079 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_professional_certification.yaml @@ -0,0 +1,4 @@ +include: _hindi_template_yaml +description: The following is multiple-choice question about Professional certification. +process_docs: !function 'utils.process_professional_certification' +task: include_base_44_hindi_professional_certification diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb237f1dec8eca933fe0bf3079ea8ab664133fc1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_social_science.yaml @@ -0,0 +1,4 @@ +include: _hindi_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_hindi_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f66b0e8a1d6846bbf96d05a0715262f9a1e20547 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/include_base_44_hindi_stem.yaml @@ -0,0 +1,4 @@ +include: _hindi_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_hindi_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hindi/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hindi/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/_hungarian_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/_hungarian_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..841556ce7ca91be4939def308095047891a6cf07 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/_hungarian_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Hungarian +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/_include_base_44_hungarian.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/_include_base_44_hungarian.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a759e8f3e9f924445c863f84d8d72d1dac43cf24 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/_include_base_44_hungarian.yaml @@ -0,0 +1,10 @@ +group: include_base_44_hungarian +task: +- include_base_44_hungarian_stem +- include_base_44_hungarian_applied_science +- include_base_44_hungarian_social_science +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/include_base_44_hungarian_applied_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/include_base_44_hungarian_applied_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec2fde5aee59218f7d9a16955d894446b636f1e5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/include_base_44_hungarian_applied_science.yaml @@ -0,0 +1,4 @@ +include: _hungarian_template_yaml +description: The following is multiple-choice question about Applied Science. +process_docs: !function 'utils.process_applied_science' +task: include_base_44_hungarian_applied_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/include_base_44_hungarian_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/include_base_44_hungarian_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7af2c1501bb88ed45370cb827bea00d5ac07aae1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/include_base_44_hungarian_social_science.yaml @@ -0,0 +1,4 @@ +include: _hungarian_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_hungarian_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/include_base_44_hungarian_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/include_base_44_hungarian_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9f818226114a9a6dd60db124351cb85b3cdcd03 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/include_base_44_hungarian_stem.yaml @@ -0,0 +1,4 @@ +include: _hungarian_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_hungarian_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Hungarian/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/_include_base_44_indonesian.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/_include_base_44_indonesian.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f854e0a769e67311043ae2ef964f60f0963c4581 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/_include_base_44_indonesian.yaml @@ -0,0 +1,12 @@ +group: include_base_44_indonesian +task: +- include_base_44_indonesian_arts_humanities +- include_base_44_indonesian_social_science +- include_base_44_indonesian_stem +- include_base_44_indonesian_applied_science +- include_base_44_indonesian_professional_certification +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/_indonesian_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/_indonesian_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..19a974c982c3e912de03cb47e55bad6d67287a4f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/_indonesian_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Indonesian +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/include_base_44_indonesian_applied_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/include_base_44_indonesian_applied_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7be50328566fd7124f415f5830f5d0c51198c964 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/include_base_44_indonesian_applied_science.yaml @@ -0,0 +1,4 @@ +include: _indonesian_template_yaml +description: The following is multiple-choice question about Applied Science. +process_docs: !function 'utils.process_applied_science' +task: include_base_44_indonesian_applied_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/include_base_44_indonesian_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/include_base_44_indonesian_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e439865e7651203a52b33760cea87c672296df88 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/include_base_44_indonesian_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _indonesian_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_indonesian_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/include_base_44_indonesian_professional_certification.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/include_base_44_indonesian_professional_certification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..960f4fd7eea40305f6bd612b6d628ebb07873bb0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/include_base_44_indonesian_professional_certification.yaml @@ -0,0 +1,4 @@ +include: _indonesian_template_yaml +description: The following is multiple-choice question about Professional certification. +process_docs: !function 'utils.process_professional_certification' +task: include_base_44_indonesian_professional_certification diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/include_base_44_indonesian_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/include_base_44_indonesian_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0bf90151fca9b16bbd4eb4a7d9d940c22a32ed3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/include_base_44_indonesian_social_science.yaml @@ -0,0 +1,4 @@ +include: _indonesian_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_indonesian_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/include_base_44_indonesian_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/include_base_44_indonesian_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6034292b721e033cd8941ff5d0c0cb18d95fbd9f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/include_base_44_indonesian_stem.yaml @@ -0,0 +1,4 @@ +include: _indonesian_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_indonesian_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Indonesian/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Italian/_include_base_44_italian.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Italian/_include_base_44_italian.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20fb8a54f63457414b8286e1a3396ad0cc9e098a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Italian/_include_base_44_italian.yaml @@ -0,0 +1,13 @@ +group: include_base_44_italian +task: +- include_base_44_italian_stem +- include_base_44_italian_arts_humanities +- include_base_44_italian_social_science +- include_base_44_italian_applied_science +- include_base_44_italian_health_oriented_education +- include_base_44_italian_professional_certification +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Italian/_italian_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Italian/_italian_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..3789e010447309a04ceb9b0789e6ceda50b05b3a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Italian/_italian_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Italian +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_applied_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_applied_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76cb047785817a0223bfec4563e54262c2350ac1 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_applied_science.yaml @@ -0,0 +1,4 @@ +include: _italian_template_yaml +description: The following is multiple-choice question about Applied Science. +process_docs: !function 'utils.process_applied_science' +task: include_base_44_italian_applied_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a88361d40b7d911c69d6abe319e055da77b87b45 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _italian_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_italian_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_health_oriented_education.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_health_oriented_education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..852d84c84747d47993ed977ab36a370e4ed203d0 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_health_oriented_education.yaml @@ -0,0 +1,4 @@ +include: _italian_template_yaml +description: The following is multiple-choice question about Health oriented education. +process_docs: !function 'utils.process_health_oriented_education' +task: include_base_44_italian_health_oriented_education diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_professional_certification.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_professional_certification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09d789465ec014de08a8d963c375b88679357301 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_professional_certification.yaml @@ -0,0 +1,4 @@ +include: _italian_template_yaml +description: The following is multiple-choice question about Professional certification. +process_docs: !function 'utils.process_professional_certification' +task: include_base_44_italian_professional_certification diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7fd9b7ffd81cac603fa42b18ce708c489bb76e94 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_social_science.yaml @@ -0,0 +1,4 @@ +include: _italian_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_italian_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..548fcd52f2efa86f13e43ccaa185e2216ee21603 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Italian/include_base_44_italian_stem.yaml @@ -0,0 +1,4 @@ +include: _italian_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_italian_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Italian/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Italian/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Italian/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Japanese/_include_base_44_japanese.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Japanese/_include_base_44_japanese.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4dddddc2204fb281a6a765ebbc2211a859523688 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Japanese/_include_base_44_japanese.yaml @@ -0,0 +1,10 @@ +group: include_base_44_japanese +task: +- include_base_44_japanese_driving_license +- include_base_44_japanese_medical_license +- include_base_44_japanese_professional_certification +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Japanese/_japanese_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Japanese/_japanese_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c8e7f87c88ae9b995ee6db5a00db4b2d8d38723 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Japanese/_japanese_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Japanese +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Japanese/include_base_44_japanese_driving_license.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Japanese/include_base_44_japanese_driving_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9245f088586fcfd78b66855d70eb6f2bb003b82a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Japanese/include_base_44_japanese_driving_license.yaml @@ -0,0 +1,4 @@ +include: _japanese_template_yaml +description: The following is multiple-choice question about Driving License. +process_docs: !function 'utils.process_driving_license' +task: include_base_44_japanese_driving_license diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Japanese/include_base_44_japanese_medical_license.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Japanese/include_base_44_japanese_medical_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ba01c5b05def3e214f8ae82b5f01eaac67b89b4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Japanese/include_base_44_japanese_medical_license.yaml @@ -0,0 +1,4 @@ +include: _japanese_template_yaml +description: The following is multiple-choice question about Medical License. +process_docs: !function 'utils.process_medical_license' +task: include_base_44_japanese_medical_license diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Japanese/include_base_44_japanese_professional_certification.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Japanese/include_base_44_japanese_professional_certification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..640e00ed62147fdd293ff04b8934be5b3309d1dc --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Japanese/include_base_44_japanese_professional_certification.yaml @@ -0,0 +1,4 @@ +include: _japanese_template_yaml +description: The following is multiple-choice question about Professional certification. +process_docs: !function 'utils.process_professional_certification' +task: include_base_44_japanese_professional_certification diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Japanese/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Japanese/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Japanese/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Kazakh/_include_base_44_kazakh.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Kazakh/_include_base_44_kazakh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7cf3c2e89280260cfca1b4493320c3a4e528a469 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Kazakh/_include_base_44_kazakh.yaml @@ -0,0 +1,8 @@ +group: include_base_44_kazakh +task: +- include_base_44_kazakh_arts_humanities +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Kazakh/_kazakh_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Kazakh/_kazakh_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..93cb8da423aa65b5d5a5461c04f1a57d8b8d4af6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Kazakh/_kazakh_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Kazakh +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Kazakh/include_base_44_kazakh_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Kazakh/include_base_44_kazakh_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8427d8a4a54c57a6a76633bbfdb2c8f6467dad2a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Kazakh/include_base_44_kazakh_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _kazakh_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_kazakh_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Kazakh/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Kazakh/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Kazakh/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Korean/_include_base_44_korean.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Korean/_include_base_44_korean.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41e491db3d0f0fd217b88e2c6ef96d4d61f01eac --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Korean/_include_base_44_korean.yaml @@ -0,0 +1,9 @@ +group: include_base_44_korean +task: +- include_base_44_korean_professional_certification +- include_base_44_korean_social_science +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Korean/_korean_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Korean/_korean_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..16600fe9f88b4558fff3319c305ce851435b2c10 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Korean/_korean_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Korean +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Korean/include_base_44_korean_professional_certification.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Korean/include_base_44_korean_professional_certification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94d360c408eed0435e10eddee0f1ae5da0f0a028 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Korean/include_base_44_korean_professional_certification.yaml @@ -0,0 +1,4 @@ +include: _korean_template_yaml +description: The following is multiple-choice question about Professional certification. +process_docs: !function 'utils.process_professional_certification' +task: include_base_44_korean_professional_certification diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Korean/include_base_44_korean_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Korean/include_base_44_korean_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ab6a7e6f2f395021b9e4da65dddc9373c85891e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Korean/include_base_44_korean_social_science.yaml @@ -0,0 +1,4 @@ +include: _korean_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_korean_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Korean/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Korean/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Korean/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/_include_base_44_lithuanian.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/_include_base_44_lithuanian.yaml new file mode 100644 index 0000000000000000000000000000000000000000..897d40bdb5ccf25efa3cec095f27164af3699f26 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/_include_base_44_lithuanian.yaml @@ -0,0 +1,12 @@ +group: include_base_44_lithuanian +task: +- include_base_44_lithuanian_arts_humanities +- include_base_44_lithuanian_stem +- include_base_44_lithuanian_social_science +- include_base_44_lithuanian_business_commerce +- include_base_44_lithuanian_professional_certification +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/_lithuanian_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/_lithuanian_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1c861798db7e4ccf5795832fbe8bb14010c9d8a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/_lithuanian_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Lithuanian +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/include_base_44_lithuanian_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/include_base_44_lithuanian_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff33d2481638bff772e90f7a3c924663730608db --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/include_base_44_lithuanian_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _lithuanian_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_lithuanian_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/include_base_44_lithuanian_business_commerce.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/include_base_44_lithuanian_business_commerce.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d43b377f2da2207af97c2a478d2af8d53d8e052 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/include_base_44_lithuanian_business_commerce.yaml @@ -0,0 +1,4 @@ +include: _lithuanian_template_yaml +description: The following is multiple-choice question about Business & Commerce. +process_docs: !function 'utils.process_business_commerce' +task: include_base_44_lithuanian_business_commerce diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/include_base_44_lithuanian_professional_certification.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/include_base_44_lithuanian_professional_certification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..643b46027f79ae9a8d7a3f6d9fe939340c5735eb --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/include_base_44_lithuanian_professional_certification.yaml @@ -0,0 +1,4 @@ +include: _lithuanian_template_yaml +description: The following is multiple-choice question about Professional certification. +process_docs: !function 'utils.process_professional_certification' +task: include_base_44_lithuanian_professional_certification diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/include_base_44_lithuanian_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/include_base_44_lithuanian_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f62b27fed7876081dfadcd88dbb96ff2784a3ec --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/include_base_44_lithuanian_social_science.yaml @@ -0,0 +1,4 @@ +include: _lithuanian_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_lithuanian_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/include_base_44_lithuanian_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/include_base_44_lithuanian_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b408a83576736cbf53d7ed511c48044dc2e9e07d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/include_base_44_lithuanian_stem.yaml @@ -0,0 +1,4 @@ +include: _lithuanian_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_lithuanian_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Lithuanian/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Malay/_include_base_44_malay.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Malay/_include_base_44_malay.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad225b54d21dc97b185d2f83b55327d96b214282 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Malay/_include_base_44_malay.yaml @@ -0,0 +1,10 @@ +group: include_base_44_malay +task: +- include_base_44_malay_social_science +- include_base_44_malay_business_commerce +- include_base_44_malay_arts_humanities +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Malay/_malay_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Malay/_malay_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d93da59ab39b1ac55e76606e2fefddec8edea16 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Malay/_malay_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Malay +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Malay/include_base_44_malay_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Malay/include_base_44_malay_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..caf255d41e606ca582ac46618c1f690e2b02bc81 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Malay/include_base_44_malay_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _malay_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_malay_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Malay/include_base_44_malay_business_commerce.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Malay/include_base_44_malay_business_commerce.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9d39eadaca8e57361cb69dc95f33677d1257d80 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Malay/include_base_44_malay_business_commerce.yaml @@ -0,0 +1,4 @@ +include: _malay_template_yaml +description: The following is multiple-choice question about Business & Commerce. +process_docs: !function 'utils.process_business_commerce' +task: include_base_44_malay_business_commerce diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Malay/include_base_44_malay_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Malay/include_base_44_malay_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..117947e10d63d758c6bd351cb53bb3004f977643 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Malay/include_base_44_malay_social_science.yaml @@ -0,0 +1,4 @@ +include: _malay_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_malay_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Malay/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Malay/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Malay/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/_include_base_44_malayalam.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/_include_base_44_malayalam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4ba552e2d2cea21751065449e7fd6961c068995 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/_include_base_44_malayalam.yaml @@ -0,0 +1,13 @@ +group: include_base_44_malayalam +task: +- include_base_44_malayalam_stem +- include_base_44_malayalam_marine_license +- include_base_44_malayalam_health_oriented_education +- include_base_44_malayalam_arts_humanities +- include_base_44_malayalam_social_science +- include_base_44_malayalam_general_knowledge +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/_malayalam_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/_malayalam_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..f21b68aa677d78193aff72900b9847ae9cb08785 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/_malayalam_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Malayalam +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df20c8e14766a71cc08582e67be9a017cb7102e4 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _malayalam_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_malayalam_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_general_knowledge.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_general_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d42ea34c70916343f1ccd3f311bac52965a4249 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_general_knowledge.yaml @@ -0,0 +1,4 @@ +include: _malayalam_template_yaml +description: The following is multiple-choice question about General knowledge. +process_docs: !function 'utils.process_general_knowledge' +task: include_base_44_malayalam_general_knowledge diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_health_oriented_education.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_health_oriented_education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0060ecf2b6c9bc8b36d9eca18618bbc3ed4c9005 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_health_oriented_education.yaml @@ -0,0 +1,4 @@ +include: _malayalam_template_yaml +description: The following is multiple-choice question about Health oriented education. +process_docs: !function 'utils.process_health_oriented_education' +task: include_base_44_malayalam_health_oriented_education diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_marine_license.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_marine_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8cb75de8cf438122948cba213219778cb0b84fb3 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_marine_license.yaml @@ -0,0 +1,4 @@ +include: _malayalam_template_yaml +description: The following is multiple-choice question about Marine License. +process_docs: !function 'utils.process_marine_license' +task: include_base_44_malayalam_marine_license diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b4faf9f4591b92c6c6029ab7e46edf27cd314db --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_social_science.yaml @@ -0,0 +1,4 @@ +include: _malayalam_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_malayalam_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90b66cdb5d58c3e22236b4e8df37b54e5df58c9d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/include_base_44_malayalam_stem.yaml @@ -0,0 +1,4 @@ +include: _malayalam_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_malayalam_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Malayalam/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Nepali/_include_base_44_nepali.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Nepali/_include_base_44_nepali.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94f6c278c72456cacc25b1ed95ad7c2983779c1a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Nepali/_include_base_44_nepali.yaml @@ -0,0 +1,9 @@ +group: include_base_44_nepali +task: +- include_base_44_nepali_driving_license +- include_base_44_nepali_professional_certification +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Nepali/_nepali_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Nepali/_nepali_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff1741ef02b54f92f043bc1741013e42804eaa1a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Nepali/_nepali_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Nepali +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Nepali/include_base_44_nepali_driving_license.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Nepali/include_base_44_nepali_driving_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9da29a7c6579d8ef4ac321f1c61246af0c73826 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Nepali/include_base_44_nepali_driving_license.yaml @@ -0,0 +1,4 @@ +include: _nepali_template_yaml +description: The following is multiple-choice question about Driving License. +process_docs: !function 'utils.process_driving_license' +task: include_base_44_nepali_driving_license diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Nepali/include_base_44_nepali_professional_certification.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Nepali/include_base_44_nepali_professional_certification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..026217ac0a7bf1b86f05a3933681682b7587f747 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Nepali/include_base_44_nepali_professional_certification.yaml @@ -0,0 +1,4 @@ +include: _nepali_template_yaml +description: The following is multiple-choice question about Professional certification. +process_docs: !function 'utils.process_professional_certification' +task: include_base_44_nepali_professional_certification diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Nepali/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Nepali/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Nepali/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/_include_base_44_north macedonian.yaml b/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/_include_base_44_north macedonian.yaml new file mode 100644 index 0000000000000000000000000000000000000000..776e3310e55ff88211c53ed2d1a2748e6fcb2f73 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/_include_base_44_north macedonian.yaml @@ -0,0 +1,11 @@ +group: include_base_44_north macedonian +task: +- include_base_44_north macedonian_arts_humanities +- include_base_44_north macedonian_stem +- include_base_44_north macedonian_business_commerce +- include_base_44_north macedonian_social_science +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/_north macedonian_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/_north macedonian_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..d43dedd2f5987ca02c2b6752f991930d86179808 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/_north macedonian_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: North Macedonian +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/include_base_44_north macedonian_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/include_base_44_north macedonian_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..079d3cc2a4e5bf4dc4674e1e603a9332d1c7037c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/include_base_44_north macedonian_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _north macedonian_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_north macedonian_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/include_base_44_north macedonian_business_commerce.yaml b/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/include_base_44_north macedonian_business_commerce.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c6529103274489eab77b7e14fa3ba1e89ac2386 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/include_base_44_north macedonian_business_commerce.yaml @@ -0,0 +1,4 @@ +include: _north macedonian_template_yaml +description: The following is multiple-choice question about Business & Commerce. +process_docs: !function 'utils.process_business_commerce' +task: include_base_44_north macedonian_business_commerce diff --git a/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/include_base_44_north macedonian_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/include_base_44_north macedonian_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3c63ac3eda6f70c1dea4b03480ebd200d560a21 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/include_base_44_north macedonian_social_science.yaml @@ -0,0 +1,4 @@ +include: _north macedonian_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_north macedonian_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/include_base_44_north macedonian_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/include_base_44_north macedonian_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4056924ad27d7d17c59f0051b1269a9e16156e03 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/include_base_44_north macedonian_stem.yaml @@ -0,0 +1,4 @@ +include: _north macedonian_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_north macedonian_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/North Macedonian/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Persian/_include_base_44_persian.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Persian/_include_base_44_persian.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5be0d3cb4f452c08b4346e774aebdf397ab6642d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Persian/_include_base_44_persian.yaml @@ -0,0 +1,12 @@ +group: include_base_44_persian +task: +- include_base_44_persian_arts_humanities +- include_base_44_persian_social_science +- include_base_44_persian_professional_certification +- include_base_44_persian_stem +- include_base_44_persian_driving_license +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Persian/_persian_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Persian/_persian_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb5deac2a6b8667ecebb82aa99282621fe3ce9e5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Persian/_persian_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Persian +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Persian/include_base_44_persian_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Persian/include_base_44_persian_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2848e8f837205e53faf04c7fab923155a1880eea --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Persian/include_base_44_persian_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _persian_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_persian_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Persian/include_base_44_persian_driving_license.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Persian/include_base_44_persian_driving_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2cafbba46c711ff5143de63a3e6a1696182e2eac --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Persian/include_base_44_persian_driving_license.yaml @@ -0,0 +1,4 @@ +include: _persian_template_yaml +description: The following is multiple-choice question about Driving License. +process_docs: !function 'utils.process_driving_license' +task: include_base_44_persian_driving_license diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Persian/include_base_44_persian_professional_certification.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Persian/include_base_44_persian_professional_certification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6dc6f1805592ce18908486a5a89b7d634fbfd794 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Persian/include_base_44_persian_professional_certification.yaml @@ -0,0 +1,4 @@ +include: _persian_template_yaml +description: The following is multiple-choice question about Professional certification. +process_docs: !function 'utils.process_professional_certification' +task: include_base_44_persian_professional_certification diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Persian/include_base_44_persian_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Persian/include_base_44_persian_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f6333c4f2c0fc120efc0c3ff683c141adc6788f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Persian/include_base_44_persian_social_science.yaml @@ -0,0 +1,4 @@ +include: _persian_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_persian_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Persian/include_base_44_persian_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Persian/include_base_44_persian_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b7ede848a080eca09a574972757cfff8de54b4b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Persian/include_base_44_persian_stem.yaml @@ -0,0 +1,4 @@ +include: _persian_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_persian_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Persian/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Persian/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Persian/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Polish/_include_base_44_polish.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Polish/_include_base_44_polish.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cdd4ce191ad39920b626667b5fb8db48a81f51f5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Polish/_include_base_44_polish.yaml @@ -0,0 +1,10 @@ +group: include_base_44_polish +task: +- include_base_44_polish_professional_certification +- include_base_44_polish_social_science +- include_base_44_polish_stem +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Polish/_polish_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Polish/_polish_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..797de3069fea6d8f73adf2dcd26f820eb231805e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Polish/_polish_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Polish +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Polish/include_base_44_polish_professional_certification.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Polish/include_base_44_polish_professional_certification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb8505e2dadd036ed03409e2f2035e49eb4bbb68 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Polish/include_base_44_polish_professional_certification.yaml @@ -0,0 +1,4 @@ +include: _polish_template_yaml +description: The following is multiple-choice question about Professional certification. +process_docs: !function 'utils.process_professional_certification' +task: include_base_44_polish_professional_certification diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Polish/include_base_44_polish_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Polish/include_base_44_polish_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22724778d8b8b1f06f6389af9df80785b8c1e1a9 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Polish/include_base_44_polish_social_science.yaml @@ -0,0 +1,4 @@ +include: _polish_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_polish_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Polish/include_base_44_polish_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Polish/include_base_44_polish_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38fe2cb007fab17bd720cd17a808450f69275a88 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Polish/include_base_44_polish_stem.yaml @@ -0,0 +1,4 @@ +include: _polish_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_polish_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Polish/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Polish/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Polish/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/_include_base_44_portuguese.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/_include_base_44_portuguese.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1140efb9d97c985f22396d89da89a72c4b9f8f2b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/_include_base_44_portuguese.yaml @@ -0,0 +1,13 @@ +group: include_base_44_portuguese +task: +- include_base_44_portuguese_stem +- include_base_44_portuguese_social_science +- include_base_44_portuguese_arts_humanities +- include_base_44_portuguese_health_oriented_education +- include_base_44_portuguese_business_commerce +- include_base_44_portuguese_applied_science +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/_portuguese_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/_portuguese_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..22905ebe8e6444d7a395ddadca388057d2b37b7e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/_portuguese_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Portuguese +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_applied_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_applied_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ea01e614dff6a7753e123ca401aea4d69e0f098 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_applied_science.yaml @@ -0,0 +1,4 @@ +include: _portuguese_template_yaml +description: The following is multiple-choice question about Applied Science. +process_docs: !function 'utils.process_applied_science' +task: include_base_44_portuguese_applied_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..011cd16fe850fc838151d29d976d7c9df0130ae2 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _portuguese_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_portuguese_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_business_commerce.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_business_commerce.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8e7f8f67ca83808c64edbdff7395ceebebc3c30 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_business_commerce.yaml @@ -0,0 +1,4 @@ +include: _portuguese_template_yaml +description: The following is multiple-choice question about Business & Commerce. +process_docs: !function 'utils.process_business_commerce' +task: include_base_44_portuguese_business_commerce diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_health_oriented_education.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_health_oriented_education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a81491e90dd64e5a34ed610eda7675397a0b730 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_health_oriented_education.yaml @@ -0,0 +1,4 @@ +include: _portuguese_template_yaml +description: The following is multiple-choice question about Health oriented education. +process_docs: !function 'utils.process_health_oriented_education' +task: include_base_44_portuguese_health_oriented_education diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd240be79a7fb31ce4b952ca0b313f0df993705b --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_social_science.yaml @@ -0,0 +1,4 @@ +include: _portuguese_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_portuguese_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7db9dd9134e54f94dac5d6fe9c4cac526580c64a --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/include_base_44_portuguese_stem.yaml @@ -0,0 +1,4 @@ +include: _portuguese_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_portuguese_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Portuguese/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Russian/_include_base_44_russian.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Russian/_include_base_44_russian.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4b36d155db795cf7ab426489e341860ca7c0d6e --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Russian/_include_base_44_russian.yaml @@ -0,0 +1,15 @@ +group: include_base_44_russian +task: +- include_base_44_russian_stem +- include_base_44_russian_health_oriented_education +- include_base_44_russian_arts_humanities +- include_base_44_russian_driving_license +- include_base_44_russian_social_science +- include_base_44_russian_business_commerce +- include_base_44_russian_marine_license +- include_base_44_russian_applied_science +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Russian/_russian_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Russian/_russian_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..42846a5d58b4e8326adbe5e1c38602bef4c5af4d --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Russian/_russian_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Russian +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_applied_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_applied_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a94f59984870089d8b3b92fd3a4a8c56806439c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_applied_science.yaml @@ -0,0 +1,4 @@ +include: _russian_template_yaml +description: The following is multiple-choice question about Applied Science. +process_docs: !function 'utils.process_applied_science' +task: include_base_44_russian_applied_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2301a96d6bda5d0530ba0958299532fb1df8afbe --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _russian_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_russian_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_business_commerce.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_business_commerce.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e848acc4f4da1463ff45fb9240cb8f6df7209378 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_business_commerce.yaml @@ -0,0 +1,4 @@ +include: _russian_template_yaml +description: The following is multiple-choice question about Business & Commerce. +process_docs: !function 'utils.process_business_commerce' +task: include_base_44_russian_business_commerce diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_driving_license.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_driving_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e67e062046804ea4f34af7ee6c525f3756af40ef --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_driving_license.yaml @@ -0,0 +1,4 @@ +include: _russian_template_yaml +description: The following is multiple-choice question about Driving License. +process_docs: !function 'utils.process_driving_license' +task: include_base_44_russian_driving_license diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_health_oriented_education.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_health_oriented_education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df168d7eb7894a38b7b1db266120a0a8b31c85ba --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_health_oriented_education.yaml @@ -0,0 +1,4 @@ +include: _russian_template_yaml +description: The following is multiple-choice question about Health oriented education. +process_docs: !function 'utils.process_health_oriented_education' +task: include_base_44_russian_health_oriented_education diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_marine_license.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_marine_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa9bde44db5408ea07582a4d0824a2421c5fa746 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_marine_license.yaml @@ -0,0 +1,4 @@ +include: _russian_template_yaml +description: The following is multiple-choice question about Marine License. +process_docs: !function 'utils.process_marine_license' +task: include_base_44_russian_marine_license diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e51af6f33d33d69d5e174f60710ae6eb3f0b68f --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_social_science.yaml @@ -0,0 +1,4 @@ +include: _russian_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_russian_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e03df575ccd835e8bcb4f5a16cd872f4b9603511 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Russian/include_base_44_russian_stem.yaml @@ -0,0 +1,4 @@ +include: _russian_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_russian_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Russian/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Russian/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Russian/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Serbian/_include_base_44_serbian.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Serbian/_include_base_44_serbian.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d183634572e578d2a1dcb0891ceacb5e71068ffd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Serbian/_include_base_44_serbian.yaml @@ -0,0 +1,10 @@ +group: include_base_44_serbian +task: +- include_base_44_serbian_stem +- include_base_44_serbian_arts_humanities +- include_base_44_serbian_social_science +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Serbian/_serbian_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Serbian/_serbian_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c883e994a9475071b3602527b04746dae665b0c --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Serbian/_serbian_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Serbian +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Serbian/include_base_44_serbian_arts_humanities.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Serbian/include_base_44_serbian_arts_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46ae9cc05e336ef96121b978e2386ad04796da18 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Serbian/include_base_44_serbian_arts_humanities.yaml @@ -0,0 +1,4 @@ +include: _serbian_template_yaml +description: The following is multiple-choice question about Arts & Humanities. +process_docs: !function 'utils.process_arts_humanities' +task: include_base_44_serbian_arts_humanities diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Serbian/include_base_44_serbian_social_science.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Serbian/include_base_44_serbian_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8aea79814e1293c6bfba700861dd139110f710a6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Serbian/include_base_44_serbian_social_science.yaml @@ -0,0 +1,4 @@ +include: _serbian_template_yaml +description: The following is multiple-choice question about Social Science. +process_docs: !function 'utils.process_social_science' +task: include_base_44_serbian_social_science diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Serbian/include_base_44_serbian_stem.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Serbian/include_base_44_serbian_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a97964c10cf061bd6e5314326f74697408f696cd --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Serbian/include_base_44_serbian_stem.yaml @@ -0,0 +1,4 @@ +include: _serbian_template_yaml +description: The following is multiple-choice question about STEM. +process_docs: !function 'utils.process_stem' +task: include_base_44_serbian_stem diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Serbian/utils.py b/babylm_eval_repo/multilingual/tasks/include/default/Serbian/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fde670c6bf543691c37830a7d87869023c70e447 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Serbian/utils.py @@ -0,0 +1,30 @@ +from functools import partial + + +CATEGORIES = [ + "Applied Science", + "Arts & Humanities", + "Business & Commerce", + "Driving License", + "General knowledge", + "Health oriented education", + "Marine License", + "Medical License", + "Professional certification", + "STEM", + "Social Science", +] + + +def process_docs(dataset, category): + return dataset.filter(lambda x: x["domain"] == category) + + +process_functions = { + f"process_{category.lower().replace(' & ', '_').replace(' ', '_')}": partial( + process_docs, category=category + ) + for category in CATEGORIES +} + +globals().update(process_functions) diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Spanish/_include_base_44_spanish.yaml b/babylm_eval_repo/multilingual/tasks/include/default/Spanish/_include_base_44_spanish.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d1b60c5eb11baf5aa820d27d3f3eafdee9d6dc6 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Spanish/_include_base_44_spanish.yaml @@ -0,0 +1,11 @@ +group: include_base_44_spanish +task: +- include_base_44_spanish_stem +- include_base_44_spanish_social_science +- include_base_44_spanish_arts_humanities +- include_base_44_spanish_health_oriented_education +aggregate_metric_list: +- metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/babylm_eval_repo/multilingual/tasks/include/default/Spanish/_spanish_template_yaml b/babylm_eval_repo/multilingual/tasks/include/default/Spanish/_spanish_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c43e86a2d1803097c873096b3f2326b28409cf5 --- /dev/null +++ b/babylm_eval_repo/multilingual/tasks/include/default/Spanish/_spanish_template_yaml @@ -0,0 +1,18 @@ +dataset_path: CohereLabs/include-base-44 +dataset_name: Spanish +test_split: test +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\n + D. {{option_d}}\nAnswer:" +doc_to_choice: + - A + - B + - C + - D +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0