diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8622e2833c24290007405dc90043f0c5b6ced7ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrixnli_yaml +task: afrixnli_swa_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1bd0829bfaf89354c5814eeffe1d4de8432fa540 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_kin.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Given the following premise and hypothesis in Kinyarwanda, identify\ + \ if the premise entails, contradicts, or is neutral towards the hypothesis. Please\ + \ respond with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_kin_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d3158d45209fe96a7e0b7520d065c9108a0798b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_twi.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Given the following premise and hypothesis in Twi, identify if the premise\ + \ entails, contradicts, or is neutral towards the hypothesis. Please respond with\ + \ exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \n\ + Hypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_twi_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51fbdc79b0987386839599a3521bb2d564256e83 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_wol.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Given the following premise and hypothesis in Wolof, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_wol_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..00ca9d17934256c4169ba0da95f7a604c60ac037 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_xho.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Given the following premise and hypothesis in isiXhosa, identify if\ + \ the premise entails, contradicts, or is neutral towards the hypothesis. Please\ + \ respond with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_xho_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yaml new file mode 100644 index 0000000000000000000000000000000000000000..04609ac3c424b323858858fdbffbd83ccec52b7e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yaml @@ -0,0 +1,30 @@ +tag: + - afrixnli_tasks + - afrixnli_tasks_prompt_3 +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d8b2f847f473ad2d1857e924495e98fffbc9edd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yor.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Given the following premise and hypothesis in Yoruba, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_yor_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83b87141b4021baad581be7d6b60375c40ef73c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_zul.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "Given the following premise and hypothesis in Zulu, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_zul_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..422ed169bffa2777cd91307c3ab097619e4d5399 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_3/utils.py @@ -0,0 +1,6 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64157b549956f932b3ad5bd2f67610378d683596 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ewe.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Ewe language.\nAnalyze the premise and hypothesis given in Ewe, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_ewe_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78da10cf7e04482eef5c0f9517a176196681c103 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_fra.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the French language.\nAnalyze the premise and hypothesis given in French, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_fra_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73fdba2fba55cc9af3bb802e50562de8ceb9a97e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ibo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Igbo language.\nAnalyze the premise and hypothesis given in Igbo, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_ibo_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f975d82b4a3b2ee0898aa8cc9aec225b3bb26e2d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_kin.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Kinyarwanda language.\nAnalyze the premise and hypothesis given in Kinyarwanda,\ + \ and determine the relationship between them.\n Respond with one of the following\ + \ options: 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_kin_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63062ac444bcd71951152416384ae5852510decb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lin.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Lingala language.\nAnalyze the premise and hypothesis given in Lingala, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_lin_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1553c620009ec7374378e584e5f7523ff6d57306 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lug.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Luganda language.\nAnalyze the premise and hypothesis given in Luganda, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_lug_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba2a377b7fb03f6cd1546fe8f1b65549d2133d6c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_orm.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Oromo language.\nAnalyze the premise and hypothesis given in Oromo, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_orm_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afce6e955b21eb514b4dfd024d7a8d115a3377ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sna.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the chiShona language.\nAnalyze the premise and hypothesis given in chiShona,\ + \ and determine the relationship between them.\n Respond with one of the following\ + \ options: 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_sna_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40c7cf8476813d29347fdd8e14785cb61a48c172 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sot.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Sesotho language.\nAnalyze the premise and hypothesis given in Sesotho, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_sot_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c28aaae79accde1d796fb5dff56f8998130df0b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_swa.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Swahili language.\nAnalyze the premise and hypothesis given in Swahili, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_swa_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9835314e7fc979010c80e6be80ebd616eb3abff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_twi.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Twi language.\nAnalyze the premise and hypothesis given in Twi, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_twi_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b535bc2d45555821810abb91755fb2afbae9bd1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_wol.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Wolof language.\nAnalyze the premise and hypothesis given in Wolof, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_wol_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45f55e0e1441fe40f0a6fcecec0309d9c3013dc3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_xho.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the isiXhosa language.\nAnalyze the premise and hypothesis given in isiXhosa,\ + \ and determine the relationship between them.\n Respond with one of the following\ + \ options: 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_xho_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe5de1a6dd271c23b14712b09ab070ec848b753b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yaml @@ -0,0 +1,30 @@ +tag: + - afrixnli_tasks + - afrixnli_tasks_prompt_4 +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63d4f60642c71a1d881af27291a73e04b4abca34 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yor.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Yoruba language.\nAnalyze the premise and hypothesis given in Yoruba, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_yor_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b4a232e395e36f6180a81247b23b624efcdbd05 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_zul.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Zulu language.\nAnalyze the premise and hypothesis given in Zulu, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_zul_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d97a0a288508e817ab695e637fb157a08c813808 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_4/utils.py @@ -0,0 +1,19 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70873a211527bec45adf7c689deef653eb3cfe07 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_amh.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_amh_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..675264a8dc0da305ec2d92ff16a8393fd9bd0729 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_eng_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f60db0bffdd4ed7c367a10c6e365707a02348a4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ewe.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_ewe_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bb558dabcb62d4e7c49c54c100986703fdc88ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_fra.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_fra_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..728ae1b805f2ac9f014200ad59c82b6e822ca884 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_hau.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_hau_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3086b9b4f3c31121e089a8f0bc4c9e9ee4c1cc4a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_ibo_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13a8845cf1ca22f4a25c79931d526a3305a2172c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_kin.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_kin_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0250f29f300f3f0d312744b0c7a83bfbcc1bc55 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lin.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_lin_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..928b74ce4fce73f952ac71999b4dbfc83c9632cb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lug.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_lug_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7f555db795996ac482deaae924db8af58e5c123 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_orm.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_orm_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac0ef3007edbcdcaf6a705202565ec1d842889a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sna.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_sna_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21fcdde5b66733a9f488c12b207545241b7ee7e6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sot.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_sot_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d5824adcf3373864aa3ecf660952f667c648ea8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_swa.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_swa_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b519ef71eec7ddf91fc2e247021779edfec29145 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_twi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_twi_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a865c8b166b19e458c3ff68138f74e48f8ce6b60 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_wol.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_wol_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1891bfd0592a8d3c6e2f0e98619bbeee234a852f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_xho.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_xho_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yaml new file mode 100644 index 0000000000000000000000000000000000000000..13e2b6ef7244d2689d7c56146aa15328e792c2fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yaml @@ -0,0 +1,30 @@ +tag: + - afrixnli_tasks + - afrixnli_tasks_prompt_5 +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "true" + - "inconclusive" + - "false" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4510441b606a8dbc0e635a00c3a009c4f891bd23 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yor.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_yor_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2aa872b0410f56ea9e7ea19c4fb3d5adf93d323d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_zul.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_zul_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6b9cb312b25a4c21bdd3d6a5e0a4e8e160451e4a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/direct/prompt_5/utils.py @@ -0,0 +1,6 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + replacements = {0: "true", 1: "false", 2: "inconclusive"} + return replacements[doc["label"]] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e8974c99a1b5d8ebed9c9be29e3628ad7d41674 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_amh diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7936a4322a3948093eefc12364f47b25181b0227 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_eng diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe2fce97e33d8958a7a064aa25baa7e86d6f8f21 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_ewe diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07c2f66238939ab19cce5f697826a5f53cdbe876 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_fra diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..885e571b344e78cf0277dc5f3193dc6096386d40 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_hau diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a8428267e752a552c6bc67baaefd4a65f1bf47f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_ibo diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..611f61df85e89324769b6065e269e48ff3902190 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_kin diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eed83c757faa913a2b220ced61eeb718c5da3c12 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_lin diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c3cc02445be42a6bdb4860b6300f59e5dbc622c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_lug diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4931dc0a9ef3a58d9e9cdca3c6ab128333f7d3a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_orm diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad7660a03668f00c0bf5a46c1162d32f382831ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_sna diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..266605cb8c0deffca5020416e45d77a444b8f313 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_sot diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07a890927933a0dc665b98f7e56cbd620fa97b18 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_swa diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d76fff819e09299df81564cc8217a2f34e20afbf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_twi diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f189d3975a07b125581d482e268205793e1577e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_wol diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..307b42fc58bf7782823473fe67a2343698c8ae9a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_xho diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yaml new file mode 100644 index 0000000000000000000000000000000000000000..3147dd0e24619dab8e927dc40ba54110b3e70c49 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yaml @@ -0,0 +1,31 @@ +tag: + - afrixnli + - afrixnli_manual_direct +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_text: !function utils.doc_to_text +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2b9f99a05509897253504e57671be3df94adaf7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_yor diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2833840644b75044470a2dfb133d0afd43da105c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_zul diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d97a0a288508e817ab695e637fb157a08c813808 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/direct/utils.py @@ -0,0 +1,19 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa79494a59c3d529cefe3afc6793c113136ba4a9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_amh diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d209484bdec5139ce18e3c84b9385cbe5549928 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_ewe diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a943963b9075e818074e98fcd3bf255502dd482a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_fra diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a343c952fe31f91f2332204df366a5434fd62f03 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_hau diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0057e6b0cbfbd7a7663829a05e0d44d60c301d3f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_ibo diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5874ae5f6cd2cac4296b3abaa5568a4dd7d2188a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_kin diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6e6023cf5ad47cede58060b973ee9aed9964bde --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_lin diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bc3a14d41eba99d4bb9f2b46fd44ec1526507cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_lug diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb9a494f4057783cca9a68eda9b4fb56e0b99948 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_orm diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6523987f10926fa2f2fd80417e86e494363f0fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_sna diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..319e909c84cf513aa9985a0a6cc44794f78a09b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_sot diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a34eb438e4d45bdedc68f893af2fb4374fc931a7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_swa diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0535f1db84f43aaed989efcfbe9e1781480931b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_twi diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8541b73ec8e1c0c7417a5547cdfd170ed9bcf21b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_wol diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d1eebcb37f65f1ad44a098220f979aa840b4f57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_xho diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yaml new file mode 100644 index 0000000000000000000000000000000000000000..089dc446943a5ca2c405e2ff71fcfdfa5cfb8b89 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yaml @@ -0,0 +1,29 @@ +tag: + - afrixnli + - afrixnli_manual_direct +dataset_path: masakhane/afrixnli-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +doc_to_text: !function utils.doc_to_text +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bf52549d730596f90caf263fe6299bbc705095b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_zul diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d97a0a288508e817ab695e637fb157a08c813808 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/lai prompt/translate/utils.py @@ -0,0 +1,19 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/afrixnli_tt.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/afrixnli_tt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba507b39d0320bc6307062fc3158a2f1d9212c84 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/afrixnli_tt.yaml @@ -0,0 +1,9 @@ +group: afrixnli_tt-irokobench +task: + - afrixnli_tt_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92ef8df7270120b2ead5d3ece0d9cffc2bfc1741 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_amh.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_amh_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa32dd72a6bd7412eae5fb94ccb3d5af06402a1c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ewe.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_ewe_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77f22faf2789d56de00b4a226832e2cb3d401362 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_hau.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_hau_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a52861402a8c3d65ccbe025fa62807d86e89b14 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_kin.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_kin_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb2a667e864afb856ec85ecd6b300378b44f8050 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lin.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_lin_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf45d957a3d4f169630b3c3405b348a0ceefe1b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lug.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_lug_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..14b20a1c35ae63adddddbc4a0b8d4e1fba2c90b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_orm.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_orm_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13103dd7a2f339f02b280dd3c67d8ec27807c86a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sna.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_sna_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..877787a88c0e8ee0c2d7d87d97aeff2d91e1330f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_wol.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_wol_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c907a2bf453d99c048822ad93feb12781004d2d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_xho.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_xho_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c0ec7c91ff7d81a500bb1d335fe06406a3c94e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yor.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_yor_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78753d1fe3ef143c547780041e70cc03d20289f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_zul.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_zul_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0810f6b37b9c83815fab4de50d3bc42b2c01624e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrixnli_translate_yaml +task: afrixnli_translate_amh_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f17a4ecf0a3fe044c04fff53ef61f5946bf744b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrixnli_translate_yaml +task: afrixnli_translate_fra_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..688778c3195c54868f0f3d1d9f56e17c167205f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrixnli_translate_yaml +task: afrixnli_translate_hau_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ssw_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ssw_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c247e565f839de423ac4aeecc79198189471d126 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ssw_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-ssw_Latn +doc_to_target: sentence_ssw_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSwati: " +include: flores +task: flores_eng_Latn-ssw_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tsn_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tsn_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d782a2af5cde5ae3a006c205a0796ea1a15750d8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tsn_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-tsn_Latn +doc_to_target: sentence_tsn_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSetswana: " +include: flores +task: flores_eng_Latn-tsn_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tum_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tum_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9036f3b7a1d2c4fa91a1f4278c1019cdf2bc68a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tum_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-tum_Latn +doc_to_target: sentence_tum_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nTumbuka: " +include: flores +task: flores_eng_Latn-tum_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-twi_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-twi_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9658615983d1c44bfd74d88def6db73a465ce96d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-twi_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-twi_Latn +doc_to_target: sentence_twi_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nTwi: " +include: flores +task: flores_eng_Latn-twi_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-umb_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-umb_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd95ac316ab006df8c4a52867ae3fdafafa36da2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-umb_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-umb_Latn +doc_to_target: sentence_umb_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nUmbundu: " +include: flores +task: flores_eng_Latn-umb_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_knc_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_knc_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37d5d624ff0f55b15649c5468f215b069efd4bcb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_knc_Arab-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: knc_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Central Kanuri\ + \ (Arabic script) sentences to English \nCentral Kanuri (Arabic script): {{sentence_knc_Arab}}\n\ + English: " +include: flores +task: flores_knc_Arab-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lin_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lin_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82543c70d5b26ed5b79288c0775a6d21216bfbe8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lin_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Lingala sentences\ + \ to English \nLingala: {{sentence_lin_Latn}}\nEnglish: " +include: flores +task: flores_lin_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lug_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lug_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb9f47bcaf95caaac8de7730dba8f662dac0230c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lug_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Luganda sentences\ + \ to English \nLuganda: {{sentence_lug_Latn}}\nEnglish: " +include: flores +task: flores_lug_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_luo_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_luo_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6000ab87662f6753b7dd98d97dc1c057c6c23b58 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_luo_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: luo_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Luo sentences\ + \ to English \nLuo: {{sentence_luo_Latn}}\nEnglish: " +include: flores +task: flores_luo_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_mos_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_mos_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b72acf36fdc39990bc8d6a91a13e1194ce3d42df --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_mos_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mos_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Mossi sentences\ + \ to English \nMossi: {{sentence_mos_Latn}}\nEnglish: " +include: flores +task: flores_mos_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nso_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nso_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..028aa75cc17d326bf4d1d85b5c96ff050bb8d78e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nso_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Northern Sotho\ + \ sentences to English \nNorthern Sotho: {{sentence_nso_Latn}}\nEnglish: " +include: flores +task: flores_nso_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nya_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nya_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5ceb01789f8d71651931a85a2b3580381895d97 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nya_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nya_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Nyanja sentences\ + \ to English \nNyanja: {{sentence_nya_Latn}}\nEnglish: " +include: flores +task: flores_nya_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_plt_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_plt_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2cdace5ed128379cd6093e6da5fee9345ef44c3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_plt_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: plt_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Plateau Malagasy\ + \ sentences to English \nPlateau Malagasy: {{sentence_plt_Latn}}\nEnglish: " +include: flores +task: flores_plt_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_run_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_run_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa4b5bc968c230b50942903d989e30e80cb51f8b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_run_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: run_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Rundi sentences\ + \ to English \nRundi: {{sentence_run_Latn}}\nEnglish: " +include: flores +task: flores_run_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sag_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sag_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b20eef56654fac2f2f086dcf6e0deea8a59c345d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sag_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sag_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Sango sentences\ + \ to English \nSango: {{sentence_sag_Latn}}\nEnglish: " +include: flores +task: flores_sag_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sna_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sna_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0c98f038617264c525edf3d0df5325e05da55ce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sna_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Shona sentences\ + \ to English \nShona: {{sentence_sna_Latn}}\nEnglish: " +include: flores +task: flores_sna_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_som_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_som_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b862c759b912e197cc16acda3cb68d1271d77e0f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_som_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Somali sentences\ + \ to English \nSomali: {{sentence_som_Latn}}\nEnglish: " +include: flores +task: flores_som_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sot_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sot_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5d4e24709a334418b7a23a5d0852f7e5ea665b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sot_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sot_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Southern Sotho\ + \ sentences to English \nSouthern Sotho: {{sentence_sot_Latn}}\nEnglish: " +include: flores +task: flores_sot_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ssw_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ssw_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ae236e5cbc21cba7724cb345d78ed20097b351b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ssw_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ssw_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Swati sentences\ + \ to English \nSwati: {{sentence_ssw_Latn}}\nEnglish: " +include: flores +task: flores_ssw_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sun_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sun_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a697a2194eaf477a40ca3f56787caaf563d2179 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sun_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sun_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Sundanese sentences\ + \ to English \nSundanese: {{sentence_sun_Latn}}\nEnglish: " +include: flores +task: flores_sun_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_swh_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_swh_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06dd9fcc0d384c4926a681e64f1c185c1111fe94 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_swh_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swh_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Swahili sentences\ + \ to English \nSwahili: {{sentence_swh_Latn}}\nEnglish: " +include: flores +task: flores_swh_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_taq_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_taq_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5380298e28c3be0c4c9ba536dfdcb685dd7356f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_taq_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: taq_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tamasheq sentences\ + \ to English \nTamasheq: {{sentence_taq_Latn}}\nEnglish: " +include: flores +task: flores_taq_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_taq_Tfng-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_taq_Tfng-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7cfb54197cdaffd83c578da681c1b5d36c9f4265 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_taq_Tfng-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: taq_Tfng-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tamasheq (Tifinagh\ + \ script) sentences to English \nTamasheq (Tifinagh script): {{sentence_taq_Tfng}}\n\ + English: " +include: flores +task: flores_taq_Tfng-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tir_Ethi-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tir_Ethi-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56607b6a6e76f921917eb4453b0851dd8a9fb415 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tir_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir_Ethi-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tigrinya sentences\ + \ to English \nTigrinya: {{sentence_tir_Ethi}}\nEnglish: " +include: flores +task: flores_tir_Ethi-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tsn_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tsn_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8d04febf4a6ebf564df918c236ede2ccc016b34 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tsn_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tsn_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Setswana sentences\ + \ to English \nSetswana: {{sentence_tsn_Latn}}\nEnglish: " +include: flores +task: flores_tsn_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tso_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tso_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c357e9df91da2e9b05faf883128ad9b81028331 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tso_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tso_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tsonga sentences\ + \ to English \nTsonga: {{sentence_tso_Latn}}\nEnglish: " +include: flores +task: flores_tso_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tum_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tum_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d70a89b24f187643ac4e93dcad084e598385207d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tum_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tum_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tumbuka sentences\ + \ to English \nTumbuka: {{sentence_tum_Latn}}\nEnglish: " +include: flores +task: flores_tum_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_twi_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_twi_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9dc957751e0c5115f4f8cb9d3bd47cfd3a66d9d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_twi_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: twi_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Twi sentences\ + \ to English \nTwi: {{sentence_twi_Latn}}\nEnglish: " +include: flores +task: flores_twi_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tzm_Tfng-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tzm_Tfng-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81f9c721e731ce51ac8cc8a8adc31225edfc3d59 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tzm_Tfng-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: tzm_Tfng-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Central Atlas\ + \ Tamazight sentences to English \nCentral Atlas Tamazight: {{sentence_tzm_Tfng}}\n\ + English: " +include: flores +task: flores_tzm_Tfng-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_wol_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_wol_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f1210fec591a63d43bd3afebd770b844ffd28a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_wol_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Wolof sentences\ + \ to English \nWolof: {{sentence_wol_Latn}}\nEnglish: " +include: flores +task: flores_wol_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_xho_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_xho_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f28e1bb3eed67659b3ac23ef9f97e6cd9c5ba7d8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_xho_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Xhosa sentences\ + \ to English \nXhosa: {{sentence_xho_Latn}}\nEnglish: " +include: flores +task: flores_xho_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_yor_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_yor_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e066592660b79c6b5e4d5c6046786a2b118e1eed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_yor_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Yoruba sentences\ + \ to English \nYoruba: {{sentence_yor_Latn}}\nEnglish: " +include: flores +task: flores_yor_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_zul_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_zul_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3b2fef466a1599ed1c5920328031176db342169 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_zul_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Zulu sentences\ + \ to English \nZulu: {{sentence_zul_Latn}}\nEnglish: " +include: flores +task: flores_zul_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-aeb_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-aeb_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0df4f642f499163c22733c8d1c7397f9949054c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-aeb_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-aeb_Arab +doc_to_target: sentence_aeb_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tunisian Arabic \nEnglish: {{sentence_eng_Latn}} \nTunisian Arabic: " +include: flores +task: flores_eng_Latn-aeb_Arab_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-amh_Ethi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-amh_Ethi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a53e8c2f24cba707d059a83dfa18d3f83791021 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-amh_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-amh_Ethi +doc_to_target: sentence_amh_Ethi +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Amharic \nEnglish: {{sentence_eng_Latn}} \nAmharic: " +include: flores +task: flores_eng_Latn-amh_Ethi_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ary_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ary_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb814d766327c71ebfd38d4ca046f2484aa3d3d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ary_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-ary_Arab +doc_to_target: sentence_ary_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Moroccan Arabic \nEnglish: {{sentence_eng_Latn}} \nMoroccan Arabic: " +include: flores +task: flores_eng_Latn-ary_Arab_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ban_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ban_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cff3c15bd4f226d62932f443416cc5e824dae612 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ban_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-ban_Latn +doc_to_target: sentence_ban_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Balinese \nEnglish: {{sentence_eng_Latn}} \nBalinese: " +include: flores +task: flores_eng_Latn-ban_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-bem_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-bem_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef6552a2b4fc29aa64cb6c3e4b3f1304260c9d76 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-bem_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-bem_Latn +doc_to_target: sentence_bem_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Bemba \nEnglish: {{sentence_eng_Latn}} \nBemba: " +include: flores +task: flores_eng_Latn-bem_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-dik_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-dik_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfcf7180903bace41e71d175a45c65ff68167344 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-dik_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-dik_Latn +doc_to_target: sentence_dik_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Southwestern Dinka \nEnglish: {{sentence_eng_Latn}} \nSouthwestern Dinka: " +include: flores +task: flores_eng_Latn-dik_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-dyu_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-dyu_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9fab72b27ebc9d9c9a80dd7b41c0d270f1114e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-dyu_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-dyu_Latn +doc_to_target: sentence_dyu_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Dyula \nEnglish: {{sentence_eng_Latn}} \nDyula: " +include: flores +task: flores_eng_Latn-dyu_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ewe_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ewe_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ecc34e50ab717b0fa3d8d6608cb952692446f89 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ewe_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-ewe_Latn +doc_to_target: sentence_ewe_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Ewe \nEnglish: {{sentence_eng_Latn}} \nEwe: " +include: flores +task: flores_eng_Latn-ewe_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fon_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fon_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed029237af79c6aaabe9942cb911a556718c014c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fon_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-fon_Latn +doc_to_target: sentence_fon_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Fon \nEnglish: {{sentence_eng_Latn}} \nFon: " +include: flores +task: flores_eng_Latn-fon_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fra_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fra_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d54e66c20d87b05dc59ee76a468f89fa5aca761 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fra_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-fra_Latn +doc_to_target: sentence_fra_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to French \nEnglish: {{sentence_eng_Latn}} \nFrench: " +include: flores +task: flores_eng_Latn-fra_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fuv_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fuv_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a821f58fa428af19d22b819428db35a52f4a6725 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fuv_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-fuv_Latn +doc_to_target: sentence_fuv_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Nigerian Fulfulde \nEnglish: {{sentence_eng_Latn}} \nNigerian Fulfulde: " +include: flores +task: flores_eng_Latn-fuv_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-gaz_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-gaz_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36fa1d6c4e1fad7f33e7182abfdca60a8df9d386 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-gaz_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-gaz_Latn +doc_to_target: sentence_gaz_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Oromo \nEnglish: {{sentence_eng_Latn}} \nOromo: " +include: flores +task: flores_eng_Latn-gaz_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-hau_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-hau_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aad0a48b3277c8150cb5679b4b8b77636d04b5c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-hau_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-hau_Latn +doc_to_target: sentence_hau_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Hausa \nEnglish: {{sentence_eng_Latn}} \nHausa: " +include: flores +task: flores_eng_Latn-hau_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ibo_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ibo_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b31e37cd4ce5a6891f7dff30ff75f05b99bdc48c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ibo_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-ibo_Latn +doc_to_target: sentence_ibo_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Igbo \nEnglish: {{sentence_eng_Latn}} \nIgbo: " +include: flores +task: flores_eng_Latn-ibo_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kab_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kab_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d6cfd8cb97ca07352c0d7927bc2476a3e9e378a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kab_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kab_Latn +doc_to_target: sentence_kab_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kabyle \nEnglish: {{sentence_eng_Latn}} \nKabyle: " +include: flores +task: flores_eng_Latn-kab_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kam_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kam_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd2da95c49b7828dfbc174d6a9d891546d433ecd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kam_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kam_Latn +doc_to_target: sentence_kam_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kamba \nEnglish: {{sentence_eng_Latn}} \nKamba: " +include: flores +task: flores_eng_Latn-kam_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kbp_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kbp_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b04cbdf144d5a9718f5a1f9ae38158952e6975e7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kbp_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kbp_Latn +doc_to_target: sentence_kbp_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kabiyè \nEnglish: {{sentence_eng_Latn}} \nKabiyè: " +include: flores +task: flores_eng_Latn-kbp_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kea_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kea_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a67cb9fef15715713918aaafe29f1147f40acda --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kea_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kea_Latn +doc_to_target: sentence_kea_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kabuverdianu \nEnglish: {{sentence_eng_Latn}} \nKabuverdianu: " +include: flores +task: flores_eng_Latn-kea_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kik_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kik_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1519f36e63c76ba57759547e91bab111c3796dcf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kik_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kik_Latn +doc_to_target: sentence_kik_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kikuyu \nEnglish: {{sentence_eng_Latn}} \nKikuyu: " +include: flores +task: flores_eng_Latn-kik_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kmb_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kmb_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..803989174a43ad7567cc321f7f7847039bc516d5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kmb_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kmb_Latn +doc_to_target: sentence_kmb_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kimbundu \nEnglish: {{sentence_eng_Latn}} \nKimbundu: " +include: flores +task: flores_eng_Latn-kmb_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-knc_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-knc_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0d262413539f659922105528184c6b1f9c74f05 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-knc_Arab.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-knc_Arab +doc_to_target: sentence_knc_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Central Kanuri (Arabic script) \nEnglish: {{sentence_eng_Latn}} \nCentral Kanuri\ + \ (Arabic script): " +include: flores +task: flores_eng_Latn-knc_Arab_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-knc_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-knc_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61ea7a2cdf03e9cd2e6fcef2abcc6e072cb5f430 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-knc_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-knc_Latn +doc_to_target: sentence_knc_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Central Kanuri (Latin script) \nEnglish: {{sentence_eng_Latn}} \nCentral Kanuri\ + \ (Latin script): " +include: flores +task: flores_eng_Latn-knc_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kon_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kon_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1967452e0032b023b48dfd3980e9d8241aed8e09 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kon_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kon_Latn +doc_to_target: sentence_kon_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kikongo \nEnglish: {{sentence_eng_Latn}} \nKikongo: " +include: flores +task: flores_eng_Latn-kon_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lin_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lin_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05e2593bdee5d218324f959277480f74db95a82b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lin_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-lin_Latn +doc_to_target: sentence_lin_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Lingala \nEnglish: {{sentence_eng_Latn}} \nLingala: " +include: flores +task: flores_eng_Latn-lin_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lua_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lua_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f4fe01e16cf1715dbf8467e9bb6fb1558f4b923 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lua_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-lua_Latn +doc_to_target: sentence_lua_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Luba-Kasai \nEnglish: {{sentence_eng_Latn}} \nLuba-Kasai: " +include: flores +task: flores_eng_Latn-lua_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lug_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lug_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0cfc35568598cd7733748a5f06fa5a1ad5c7c85e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lug_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-lug_Latn +doc_to_target: sentence_lug_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Luganda \nEnglish: {{sentence_eng_Latn}} \nLuganda: " +include: flores +task: flores_eng_Latn-lug_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-luo_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-luo_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05c027bb0256d1a22e1c14ca2812b6f9abb65fb7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-luo_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-luo_Latn +doc_to_target: sentence_luo_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Luo \nEnglish: {{sentence_eng_Latn}} \nLuo: " +include: flores +task: flores_eng_Latn-luo_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-mos_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-mos_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a676522a51603f951c5dfe0d88a6d99823b46eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-mos_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-mos_Latn +doc_to_target: sentence_mos_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Mossi \nEnglish: {{sentence_eng_Latn}} \nMossi: " +include: flores +task: flores_eng_Latn-mos_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nso_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nso_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c681b492c17f5e95709c9f0bd06637b10c07c9c3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nso_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-nso_Latn +doc_to_target: sentence_nso_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Northern Sotho \nEnglish: {{sentence_eng_Latn}} \nNorthern Sotho: " +include: flores +task: flores_eng_Latn-nso_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nya_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nya_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..135029028e124537ec4b2dab4222fcb582d38beb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nya_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-nya_Latn +doc_to_target: sentence_nya_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Nyanja \nEnglish: {{sentence_eng_Latn}} \nNyanja: " +include: flores +task: flores_eng_Latn-nya_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sag_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sag_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32f399391b905b77d5bea93229fc6b4c5de9e533 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sag_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-sag_Latn +doc_to_target: sentence_sag_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Sango \nEnglish: {{sentence_eng_Latn}} \nSango: " +include: flores +task: flores_eng_Latn-sag_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sna_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sna_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e219c40275fb7938cc2a121822b534776aff57b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sna_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-sna_Latn +doc_to_target: sentence_sna_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Shona \nEnglish: {{sentence_eng_Latn}} \nShona: " +include: flores +task: flores_eng_Latn-sna_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sot_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sot_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..674d162b64e72d5c3d58521643a8dae6042b9cf5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sot_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-sot_Latn +doc_to_target: sentence_sot_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Southern Sotho \nEnglish: {{sentence_eng_Latn}} \nSouthern Sotho: " +include: flores +task: flores_eng_Latn-sot_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ssw_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ssw_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23b9216f912ba5cd340181dd5c07b19c4ff03c7d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ssw_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-ssw_Latn +doc_to_target: sentence_ssw_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Swati \nEnglish: {{sentence_eng_Latn}} \nSwati: " +include: flores +task: flores_eng_Latn-ssw_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-taq_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-taq_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b09b52f46c56e15fc30aff90cbac8c8b8f8e2b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-taq_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-taq_Latn +doc_to_target: sentence_taq_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tamasheq \nEnglish: {{sentence_eng_Latn}} \nTamasheq: " +include: flores +task: flores_eng_Latn-taq_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tso_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tso_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d027a2aa2fc08aaa9fa792391eb69d46ddee802 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tso_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-tso_Latn +doc_to_target: sentence_tso_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tsonga \nEnglish: {{sentence_eng_Latn}} \nTsonga: " +include: flores +task: flores_eng_Latn-tso_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores new file mode 100644 index 0000000000000000000000000000000000000000..60bf41116e43ccdd17efcdcbe0e72c8aad0cf684 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores @@ -0,0 +1,27 @@ +tag: +- african_flores_tasks +- flores_afr-eng +- flores_afr-eng_prompt_3 +- afrobench_MT_tasks +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "**" + - + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ace_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ace_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee5f12704a3a7f03a52aba093e01e335a98729ff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ace_Arab-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ace_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Acehnese (Arabic script) and English linguist, translate the following\ + \ Acehnese (Arabic script) sentences to English \nAcehnese (Arabic script): {{sentence_ace_Arab}}\n\ + English: " +include: flores +task: flores_ace_Arab-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ace_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ace_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1d70ba341b8e30123cfd2885f570e5050c359a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ace_Latn-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ace_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Acehnese (Latin script) and English linguist, translate the following\ + \ Acehnese (Latin script) sentences to English \nAcehnese (Latin script): {{sentence_ace_Latn}}\n\ + English: " +include: flores +task: flores_ace_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_acq_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_acq_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8cda39626e72a3df883f1b74c314e8c275fa4fbb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_acq_Arab-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: acq_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Ta’izzi-Adeni Arabic and English linguist, translate the following\ + \ Ta’izzi-Adeni Arabic sentences to English \nTa’izzi-Adeni Arabic: {{sentence_acq_Arab}}\n\ + English: " +include: flores +task: flores_acq_Arab-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_aeb_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_aeb_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97f8ef2c91bd0255f2b887ff6b1acb75fc1c0487 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_aeb_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aeb_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tunisian Arabic and English linguist, translate the following Tunisian\ + \ Arabic sentences to English \nTunisian Arabic: {{sentence_aeb_Arab}}\nEnglish: " +include: flores +task: flores_aeb_Arab-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_afr_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_afr_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e228cb9c66858d173835016566cd1f4731038120 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_afr_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Afrikaans and English linguist, translate the following Afrikaans\ + \ sentences to English \nAfrikaans: {{sentence_afr_Latn}}\nEnglish: " +include: flores +task: flores_afr_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_aka_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_aka_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d6fc38582c415828023478f33f2925427a68cbb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_aka_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aka_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Akan and English linguist, translate the following Akan sentences\ + \ to English \nAkan: {{sentence_aka_Latn}}\nEnglish: " +include: flores +task: flores_aka_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_amh_Ethi-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_amh_Ethi-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58f33f9a13c5c9840fd4dcdcdd12c664daf60878 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_amh_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Amharic and English linguist, translate the following Amharic sentences\ + \ to English \nAmharic: {{sentence_amh_Ethi}}\nEnglish: " +include: flores +task: flores_amh_Ethi-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ary_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ary_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3006ebf72c0088340346a3a7b8f140da84211048 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ary_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ary_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Moroccan Arabic and English linguist, translate the following Moroccan\ + \ Arabic sentences to English \nMoroccan Arabic: {{sentence_ary_Arab}}\nEnglish: " +include: flores +task: flores_ary_Arab-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_arz_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_arz_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46cc0a18d4633b7032c7c179606845941f595e8b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_arz_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arz_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Egyptian Arabic and English linguist, translate the following Egyptian\ + \ Arabic sentences to English \nEgyptian Arabic: {{sentence_arz_Arab}}\nEnglish: " +include: flores +task: flores_arz_Arab-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_bam_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_bam_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c50a8dfa4ae3b2a99c2bb40f64749fb22c8928ac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_bam_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bam_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Bambara and English linguist, translate the following Bambara sentences\ + \ to English \nBambara: {{sentence_bam_Latn}}\nEnglish: " +include: flores +task: flores_bam_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ban_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ban_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86f2eed3fef3b2aefef9f0a7e640310d054e3fc9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ban_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ban_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Balinese and English linguist, translate the following Balinese\ + \ sentences to English \nBalinese: {{sentence_ban_Latn}}\nEnglish: " +include: flores +task: flores_ban_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_bem_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_bem_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55c32fe9c5e3f4321b6c3145d862d23f48da233b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_bem_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Bemba and English linguist, translate the following Bemba sentences\ + \ to English \nBemba: {{sentence_bem_Latn}}\nEnglish: " +include: flores +task: flores_bem_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_cjk_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_cjk_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..642cd4dda88f9c7e38092fbc650e8340ff8998a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_cjk_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: cjk_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Chokwe and English linguist, translate the following Chokwe sentences\ + \ to English \nChokwe: {{sentence_cjk_Latn}}\nEnglish: " +include: flores +task: flores_cjk_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_dik_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_dik_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8005a642241e9ba4e9255a224f6c7d641553edd7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_dik_Latn-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: dik_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Southwestern Dinka and English linguist, translate the following\ + \ Southwestern Dinka sentences to English \nSouthwestern Dinka: {{sentence_dik_Latn}}\n\ + English: " +include: flores +task: flores_dik_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_dyu_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_dyu_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a99efc0867c365186db75a8f04a0fbfa741f91c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_dyu_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: dyu_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Dyula and English linguist, translate the following Dyula sentences\ + \ to English \nDyula: {{sentence_dyu_Latn}}\nEnglish: " +include: flores +task: flores_dyu_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ewe_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ewe_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77133ad60cc57992882213908c1abe2300fae291 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ewe_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Ewe and English linguist, translate the following Ewe sentences\ + \ to English \nEwe: {{sentence_ewe_Latn}}\nEnglish: " +include: flores +task: flores_ewe_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fon_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fon_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..436bf4ac3e5319e8cf2da5791607f8d4f7564eca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fon_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fon_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Fon and English linguist, translate the following Fon sentences\ + \ to English \nFon: {{sentence_fon_Latn}}\nEnglish: " +include: flores +task: flores_fon_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fra_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fra_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b10c46e3226342c3a01c90f881df8575049eb6b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fra_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a French and English linguist, translate the following French sentences\ + \ to English \nFrench: {{sentence_fra_Latn}}\nEnglish: " +include: flores +task: flores_fra_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fuv_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fuv_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ffcbd3c04f1f6fd608e11286be4d88c079890a88 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fuv_Latn-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: fuv_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Nigerian Fulfulde and English linguist, translate the following\ + \ Nigerian Fulfulde sentences to English \nNigerian Fulfulde: {{sentence_fuv_Latn}}\n\ + English: " +include: flores +task: flores_fuv_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_gaz_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_gaz_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..703cd3517a81e172683fb43b91ddbb4ca7db500e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_gaz_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: gaz_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Oromo and English linguist, translate the following Oromo sentences\ + \ to English \nOromo: {{sentence_gaz_Latn}}\nEnglish: " +include: flores +task: flores_gaz_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_hau_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_hau_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7527bf78ebc88167a99707eb3101b1c350e5c991 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_hau_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Hausa and English linguist, translate the following Hausa sentences\ + \ to English \nHausa: {{sentence_hau_Latn}}\nEnglish: " +include: flores +task: flores_hau_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ibo_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ibo_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7705911a67b2584a9d1afc3cd5c4294a37a22ece --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ibo_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Igbo and English linguist, translate the following Igbo sentences\ + \ to English \nIgbo: {{sentence_ibo_Latn}}\nEnglish: " +include: flores +task: flores_ibo_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kab_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kab_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec406c5e0fbf8f5b41b17e432586c00f8383eabd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kab_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kab_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kabyle and English linguist, translate the following Kabyle sentences\ + \ to English \nKabyle: {{sentence_kab_Latn}}\nEnglish: " +include: flores +task: flores_kab_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kam_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kam_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed27b6d79c71b5c1f4690cd409a86aabf9901124 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kam_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kam_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kamba and English linguist, translate the following Kamba sentences\ + \ to English \nKamba: {{sentence_kam_Latn}}\nEnglish: " +include: flores +task: flores_kam_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kbp_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kbp_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c1a0961e08908cf1e846a87e7ddce3641e80ead --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kbp_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kbp_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kabiyè and English linguist, translate the following Kabiyè sentences\ + \ to English \nKabiyè: {{sentence_kbp_Latn}}\nEnglish: " +include: flores +task: flores_kbp_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kea_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kea_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..67dd9e73fa327338fecec80728ac645f78996b92 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kea_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kea_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kabuverdianu and English linguist, translate the following Kabuverdianu\ + \ sentences to English \nKabuverdianu: {{sentence_kea_Latn}}\nEnglish: " +include: flores +task: flores_kea_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kmb_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kmb_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c31ede4dfe8449d2a1c8e84b74eeee0fdc908b78 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kmb_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kmb_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kimbundu and English linguist, translate the following Kimbundu\ + \ sentences to English \nKimbundu: {{sentence_kmb_Latn}}\nEnglish: " +include: flores +task: flores_kmb_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kon_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kon_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54ede3a6e2c0280761a3af529cc0a8fe82d2f518 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kon_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kon_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kikongo and English linguist, translate the following Kikongo sentences\ + \ to English \nKikongo: {{sentence_kon_Latn}}\nEnglish: " +include: flores +task: flores_kon_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores new file mode 100644 index 0000000000000000000000000000000000000000..ac7dc1651e4729ae0357c6d958745400ddc35ea1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores @@ -0,0 +1,27 @@ +tag: +- african_flores_tasks +- flores_eng-afr +- flores_eng-afr_prompt_3 +- afrobench_MT_tasks +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "**" + - + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/flores b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/flores new file mode 100644 index 0000000000000000000000000000000000000000..74f9f33eb22662bec79709bd64d8d31f3fb8eae0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/flores/prompt_3/flores @@ -0,0 +1,24 @@ +tag: +- flores_tasks +- flores_afr-eng +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner new file mode 100644 index 0000000000000000000000000000000000000000..7f32f86b1e194826a7ffe7d4edb0935eac80c491 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner @@ -0,0 +1,26 @@ +tag: +- masakhaner_tasks +- masakhaner_prompt_3 +dataset_path: masakhane/masakhaner-x +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +generation_kwargs: + do_sample: false + until: + - + - <|im_end|> +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: target +filter_list: + - name: flexible-extract + filter: + - function: format_span +metric_list: + - metric: f1 + aggregation: !function utils.span_f1_agg + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_sw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb3d69595796755c03f4f02f201d36e121e4b6bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_sw.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: sw +doc_to_text: "You are a Named Entity Recognition expert in Kiswahili language. \n\ + Extract all named entities from the following Kiswahili text and categorize them\ + \ into PERSON, LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows\ + \ the format: label: entity $$ label: entity, with each unique entity on a separate\ + \ label line, avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant\ + \ entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_sw_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_wo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_wo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6db45e2bccb590f2586494d3fb586b3e5966a17b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_wo.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: wo +doc_to_text: "You are a Named Entity Recognition expert in Wolof language. \nExtract\ + \ all named entities from the following Wolof text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_wo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_yo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_yo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..589cd5b35a6b27c8a9075ee22b9f8fd98a11860d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_yo.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: yo +doc_to_text: "You are a Named Entity Recognition expert in Yoruba language. \nExtract\ + \ all named entities from the following Yoruba text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_yo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_zu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_zu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c25d5a0c89da79ea982a35f8d032c3c489a16a80 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_zu.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: zu +doc_to_text: "You are a Named Entity Recognition expert in isiZulu language. \nExtract\ + \ all named entities from the following isiZulu text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_zu_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_mos.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_mos.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2eb75d8e554dba222cf9f92fc6c8f013e7d232b2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_mos.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: mos +doc_to_text: "As a Mossi linguist, label all named entities in the Mossi text below\ + \ with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_mos_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_pcm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_pcm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93f8ae3adb27f74df18c13a0ed886d176b62eead --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_pcm.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "As a Nigerian Pidgin linguist, label all named entities in the Nigerian\ + \ Pidgin text below with the categories: PERSON, LOCATION, ORGANIZATION, and DATE.\ + \ Ensure the output strictly follows the format: label: entity $$ label: entity,\ + \ with each unique entity on a separate label line, avoiding grouped entities (e.g.,\ + \ avoid LOC: entity, entity) or irrelevant entries like none. Return only the output.\ + \ \n\nText: {{text}}" +include: masakhaner +task: masakhaner_pcm_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_rw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_rw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d64d49925bf1668e0f2b5ebdb6516f4ba0668f88 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_rw.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: rw +doc_to_text: "As a Kinyarwanda linguist, label all named entities in the Kinyarwanda\ + \ text below with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure\ + \ the output strictly follows the format: label: entity $$ label: entity, with each\ + \ unique entity on a separate label line, avoiding grouped entities (e.g., avoid\ + \ LOC: entity, entity) or irrelevant entries like none. Return only the output.\ + \ \n\nText: {{text}}" +include: masakhaner +task: masakhaner_rw_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_tw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_tw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d2eec6befd2442e314abba2e2655dc8aa0baf4a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_tw.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: tw +doc_to_text: "As a Twi linguist, label all named entities in the Twi text below with\ + \ the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_tw_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_wo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_wo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41501cb385fe50b38d5d30afe592da4705585881 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_wo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: wo +doc_to_text: "As a Wolof linguist, label all named entities in the Wolof text below\ + \ with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_wo_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_yo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_yo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0c327bd53c90651e9c7e6b699d3fb9fb52748a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_yo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: yo +doc_to_text: "As a Yoruba linguist, label all named entities in the Yoruba text below\ + \ with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_yo_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_zu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_zu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24961ec759862a77b2ce608f4a5954ec62f139fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_zu.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: zu +doc_to_text: "As a isiZulu linguist, label all named entities in the isiZulu text\ + \ below with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the\ + \ output strictly follows the format: label: entity $$ label: entity, with each\ + \ unique entity on a separate label line, avoiding grouped entities (e.g., avoid\ + \ LOC: entity, entity) or irrelevant entries like none. Return only the output.\ + \ \n\nText: {{text}}" +include: masakhaner +task: masakhaner_zu_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..76909044e7f35948156f8bb506ce2fce563ec689 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_4/utils.py @@ -0,0 +1,146 @@ +import collections +import re + +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + return transform_text(doc["ner_tags"]) + + +def transform_text(text): + entities = [] + current_entity = "" + current_tag = "" + + for pair in text.split("\n"): + if pair: # Check if the line is not empty + word, tag = pair.strip().split(": ") + tag = tag.upper() + word = word.lower() + word = word.strip(",.").strip() + + if tag.startswith("B-"): + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + current_tag = tag.split("-")[1] + current_entity = word + elif tag.startswith("I-") and tag.split("-")[1] == current_tag: + current_entity += word + else: + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + current_entity = "" + current_tag = "" + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + + # Join all the transformed output lines with $$ as separator + return " $$ ".join(entities) + + +def span_f1_agg(items): + """Computes Span based F1 score. + + This function is copied from + https://github.com/google-research/multilingual-t5/blob/master/multilingual_t5/evaluation/metrics.py + + Args: + targets: list of strings or list of list of strings if multiple references + are present. + predictions: list of strings + + Returns: + span f1 across all targets and predictions (Based on CoNLL script) + """ + unzipped_list = list(zip(*items)) + targets = unzipped_list[0] + predictions = unzipped_list[1] + + true_positives = collections.defaultdict(int) + false_positives = collections.defaultdict(int) + false_negatives = collections.defaultdict(int) + + def normalize_text(strings): + def get_blank_spaces_pattern(): + return re.compile(r"\s{3,}|\t") + + def remove_blank_spaces(text): + text = re.sub(pattern=get_blank_spaces_pattern(), repl="", string=text) + text = re.sub("\s+", " ", text) + return text + + def remove_punctuation(text): + my_punctuation = '!"$%&\'()*+,-./:;<=>?[\\]^_`{|}~•@.""-,`' + text = re.sub( + "[" + my_punctuation + "]+", " ", str(text) + ) # strip punctuation + return text + + def remove_articles(text): + regex = re.compile(r"\b(a|an|the)\b", re.UNICODE) + return re.sub(regex, " ", text) + + def lowercase(text): + text = text.lower() + return text + + strings = remove_punctuation(strings) + strings = remove_articles(strings) + strings = remove_blank_spaces(strings) + strings = lowercase(strings) + + return strings + + def tags_to_spans(tag_sequence, delimiter="$$"): + """Extract spans from IOB1 or BIO tags.""" + if isinstance(tag_sequence, list): + tag_sequence = " ".join(i.strip() for i in tag_sequence) + tag_sequence_split = [ + item.strip() + for sub in tag_sequence.strip().split(delimiter) + for item in sub.split("$") + if item + ] + tag_sequence_split = [ + item.strip() + for value in tag_sequence_split + for sub in value.split(". ") + for item in sub.split(", ") + ] + tags_entities = [] + for tag_entity in tag_sequence_split: + tag_entity_split = tag_entity.split(": ") + if len(tag_entity_split) != 2: + continue + tag = normalize_text(tag_entity_split[0].strip()) + entity = normalize_text(tag_entity_split[1].rstrip().lstrip()) + tags_entities.append((tag, entity)) + return tags_entities + + def compute_f1_metrics(true_positive, false_positive, false_negative): + precision = float(true_positive) / float(true_positive + false_positive + 1e-13) + recall = float(true_positive) / float(true_positive + false_negative + 1e-13) + f1_measures = 2.0 * ((precision * recall) / (precision + recall + 1e-13)) + return precision, recall, f1_measures + + for target, pred in zip(targets, predictions): + gold_spans = tags_to_spans(target) + predicted_spans = tags_to_spans(pred) + + for span in predicted_spans: + if span in gold_spans: + true_positives[span[0]] += 1 + gold_spans.remove(span) + else: + false_positives[span[0]] += 1 + # These spans weren't predicted. + for span in gold_spans: + false_negatives[span[0]] += 1 + + _, _, f1_measure = compute_f1_metrics( + sum(true_positives.values()), + sum(false_positives.values()), + sum(false_negatives.values()), + ) + return f1_measure diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_am.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_am.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90c485745377aa9532eb0f6e7b35b15cd31d5414 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_am.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: am +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_am_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_bbj.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_bbj.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74726694ef67fd5f572c1a58b0b637cf410a9997 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_bbj.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_bbj_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ig.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ig.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b8a429593210e2db50535011174cdbff26ad9c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ig.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: ig +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ig_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_lg.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_lg.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84bdc8b9af8e76d09da546db020390531450ed85 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_lg.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: lg +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_lg_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55a0b5744cdaa4cf894ac882642978f66c7dbe5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_luo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: luo +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_luo_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_mos.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_mos.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06bcc4467d43d8559b8e9b2cd4b0a89fa6b3fa40 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_mos.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: mos +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_mos_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_pcm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_pcm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9d897bc01a217c01d9fbffdf496d060ec54b434 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_pcm.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_pcm_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_rw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_rw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0742bc4dd567005a51861bab9b6208d64c66166f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_rw.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: rw +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_rw_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_sn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_sn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56711335c8c11879338fbc3b245bc49d0702733e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_sn.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: sn +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_sn_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_sw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c418beb45612e31ff1738909d5d1c181bfbab079 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_sw.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: sw +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_sw_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_tn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_tn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf94a1081352827a7fe09eb913bfabd9e5f0c576 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_tn.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: tn +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_tn_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_tw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_tw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cad2e2e3e64819dc6ab3151929a6ec76bb868821 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_tw.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: tw +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_tw_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_wo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_wo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec7af039234047cda3500be81b361bae294bace2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_wo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: wo +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_wo_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_xh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_xh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..debb164aef161a178ca53046e9b674a677d5fc08 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_xh.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: xh +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_xh_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_yo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_yo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9abe1acbcb473c1bd091277c8a1913c792fdd0a3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_yo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: yo +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_yo_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_zu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_zu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5af591aa464ef88ff6a4d4b62e55c504cb777c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_zu.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: zu +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_zu_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..76909044e7f35948156f8bb506ce2fce563ec689 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhaner/prompt_5/utils.py @@ -0,0 +1,146 @@ +import collections +import re + +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + return transform_text(doc["ner_tags"]) + + +def transform_text(text): + entities = [] + current_entity = "" + current_tag = "" + + for pair in text.split("\n"): + if pair: # Check if the line is not empty + word, tag = pair.strip().split(": ") + tag = tag.upper() + word = word.lower() + word = word.strip(",.").strip() + + if tag.startswith("B-"): + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + current_tag = tag.split("-")[1] + current_entity = word + elif tag.startswith("I-") and tag.split("-")[1] == current_tag: + current_entity += word + else: + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + current_entity = "" + current_tag = "" + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + + # Join all the transformed output lines with $$ as separator + return " $$ ".join(entities) + + +def span_f1_agg(items): + """Computes Span based F1 score. + + This function is copied from + https://github.com/google-research/multilingual-t5/blob/master/multilingual_t5/evaluation/metrics.py + + Args: + targets: list of strings or list of list of strings if multiple references + are present. + predictions: list of strings + + Returns: + span f1 across all targets and predictions (Based on CoNLL script) + """ + unzipped_list = list(zip(*items)) + targets = unzipped_list[0] + predictions = unzipped_list[1] + + true_positives = collections.defaultdict(int) + false_positives = collections.defaultdict(int) + false_negatives = collections.defaultdict(int) + + def normalize_text(strings): + def get_blank_spaces_pattern(): + return re.compile(r"\s{3,}|\t") + + def remove_blank_spaces(text): + text = re.sub(pattern=get_blank_spaces_pattern(), repl="", string=text) + text = re.sub("\s+", " ", text) + return text + + def remove_punctuation(text): + my_punctuation = '!"$%&\'()*+,-./:;<=>?[\\]^_`{|}~•@.""-,`' + text = re.sub( + "[" + my_punctuation + "]+", " ", str(text) + ) # strip punctuation + return text + + def remove_articles(text): + regex = re.compile(r"\b(a|an|the)\b", re.UNICODE) + return re.sub(regex, " ", text) + + def lowercase(text): + text = text.lower() + return text + + strings = remove_punctuation(strings) + strings = remove_articles(strings) + strings = remove_blank_spaces(strings) + strings = lowercase(strings) + + return strings + + def tags_to_spans(tag_sequence, delimiter="$$"): + """Extract spans from IOB1 or BIO tags.""" + if isinstance(tag_sequence, list): + tag_sequence = " ".join(i.strip() for i in tag_sequence) + tag_sequence_split = [ + item.strip() + for sub in tag_sequence.strip().split(delimiter) + for item in sub.split("$") + if item + ] + tag_sequence_split = [ + item.strip() + for value in tag_sequence_split + for sub in value.split(". ") + for item in sub.split(", ") + ] + tags_entities = [] + for tag_entity in tag_sequence_split: + tag_entity_split = tag_entity.split(": ") + if len(tag_entity_split) != 2: + continue + tag = normalize_text(tag_entity_split[0].strip()) + entity = normalize_text(tag_entity_split[1].rstrip().lstrip()) + tags_entities.append((tag, entity)) + return tags_entities + + def compute_f1_metrics(true_positive, false_positive, false_negative): + precision = float(true_positive) / float(true_positive + false_positive + 1e-13) + recall = float(true_positive) / float(true_positive + false_negative + 1e-13) + f1_measures = 2.0 * ((precision * recall) / (precision + recall + 1e-13)) + return precision, recall, f1_measures + + for target, pred in zip(targets, predictions): + gold_spans = tags_to_spans(target) + predicted_spans = tags_to_spans(pred) + + for span in predicted_spans: + if span in gold_spans: + true_positives[span[0]] += 1 + gold_spans.remove(span) + else: + false_positives[span[0]] += 1 + # These spans weren't predicted. + for span in gold_spans: + false_negatives[span[0]] += 1 + + _, _, f1_measure = compute_f1_metrics( + sum(true_positives.values()), + sum(false_positives.values()), + sum(false_negatives.values()), + ) + return f1_measure diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/README.md b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/README.md new file mode 100644 index 0000000000000000000000000000000000000000..16df2df1d62f2d83d6d34e22373d6680a246eaa8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/README.md @@ -0,0 +1,99 @@ +# + +## Paper +Title: `MasakhaNEWS: News Topic Classification for African languages` + +Paper Link: https://aclanthology.org/2023.ijcnlp-main.10/ + +## Abstract +>African languages are severely under-represented in NLP research due to lack of datasets covering several NLP tasks. While there are individual language specific datasets that are being expanded to different tasks, only a handful of NLP tasks (e.g. named entity recognition and machine translation) have standardized benchmark datasets covering several geographical and typologically-diverse African languages. In this paper, we develop MasakhaNEWS -- a new benchmark dataset for news topic classification covering 16 languages widely spoken in Africa. We provide an evaluation of baseline models by training classical machine learning models and fine-tuning several language models. Furthermore, we explore several alternatives to full fine-tuning of language models that are better suited for zero-shot and few-shot learning such as cross-lingual parameter-efficient fine-tuning (like MAD-X), pattern exploiting training (PET), prompting language models (like ChatGPT), and prompt-free sentence transformer fine-tuning (SetFit and Cohere Embedding API). Our evaluation in zero-shot setting shows the potential of prompting ChatGPT for news topic classification in low-resource African languages, achieving an average performance of 70 F1 points without leveraging additional supervision like MAD-X. In few-shot setting, we show that with as little as 10 examples per label, we achieved more than 90% (i.e. 86.0 F1 points) of the performance of full supervised training (92.6 F1 points) leveraging the PET approach. + +HomePage: https://github.com/masakhane-io/masakhane-news + +### Citation + +``` +@inproceedings{adelani-etal-2023-masakhanews, + title = "{M}asakha{NEWS}: News Topic Classification for {A}frican languages", + author = "Adelani, David Ifeoluwa and + Masiak, Marek and + Azime, Israel Abebe and + Alabi, Jesujoba and + Tonja, Atnafu Lambebo and + Mwase, Christine and + Ogundepo, Odunayo and + Dossou, Bonaventure F. P. and + Oladipo, Akintunde and + Nixdorf, Doreen and + Emezue, Chris Chinenye and + Al-azzawi, Sana and + Sibanda, Blessing and + David, Davis and + Ndolela, Lolwethu and + Mukiibi, Jonathan and + Ajayi, Tunde and + Moteu, Tatiana and + Odhiambo, Brian and + Owodunni, Abraham and + Obiefuna, Nnaemeka and + Mohamed, Muhidin and + Muhammad, Shamsuddeen Hassan and + Ababu, Teshome Mulugeta and + Salahudeen, Saheed Abdullahi and + Yigezu, Mesay Gemeda and + Gwadabe, Tajuddeen and + Abdulmumin, Idris and + Taye, Mahlet and + Awoyomi, Oluwabusayo and + Shode, Iyanuoluwa and + Adelani, Tolulope and + Abdulganiyu, Habiba and + Omotayo, Abdul-Hakeem and + Adeeko, Adetola and + Afolabi, Abeeb and + Aremu, Anuoluwapo and + Samuel, Olanrewaju and + Siro, Clemencia and + Kimotho, Wangari and + Ogbu, Onyekachi and + Mbonu, Chinedu and + Chukwuneke, Chiamaka and + Fanijo, Samuel and + Ojo, Jessica and + Awosan, Oyinkansola and + Kebede, Tadesse and + Sakayo, Toadoum Sari and + Nyatsine, Pamela and + Sidume, Freedmore and + Yousuf, Oreen and + Oduwole, Mardiyyah and + Tshinu, Kanda and + Kimanuka, Ussen and + Diko, Thina and + Nxakama, Siyanda and + Nigusse, Sinodos and + Johar, Abdulmejid and + Mohamed, Shafie and + Hassan, Fuad Mire and + Mehamed, Moges Ahmed and + Ngabire, Evrard and + Jules, Jules and + Ssenkungu, Ivan and + Stenetorp, Pontus", + editor = "Park, Jong C. and + Arase, Yuki and + Hu, Baotian and + Lu, Wei and + Wijaya, Derry and + Purwarianti, Ayu and + Krisnadhi, Adila Alfa", + booktitle = "Proceedings of the 13th International Joint Conference on Natural Language Processing and the 3rd Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = nov, + year = "2023", + address = "Nusa Dua, Bali", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2023.ijcnlp-main.10/", + doi = "10.18653/v1/2023.ijcnlp-main.10", + pages = "144--159" +} +``` diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/masakhanews.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/masakhanews.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93b6f29d8cdc05cf0904e4a8fe9afdd35d111c88 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/masakhanews.yaml @@ -0,0 +1,13 @@ +group: masakhanews +task: + - masakhanews_prompt_1 + - masakhanews_prompt_2 + - masakhanews_prompt_3 + - masakhanews_prompt_4 + - masakhanews_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d45b784facda2e60335d1980b9a1038c5cb91ec0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_amh_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40685c17d8616dafb3797ab84153f77242e6a364 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_eng_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2371172156b0cfb8532d93366518f2faf8793aed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_fra.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_fra_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7288982d35eaaad09407c59b6b6d02af3fb637a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_hau_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bf65cca861c670be75aa9b63a914590d4d996c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_ibo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6cdbe8de5ef1593bd2f98d75ec4ecca3fc33084 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_lin_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2f0ec1bada04bf7c72649621330e828bd449951 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_lug_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9bff1ac5113f06c5d035e46c0e139fbeb0a8d28 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_orm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_orm_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_pcm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_pcm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..119b01bb158d15d307949226dc71a055d169e4fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_pcm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_pcm_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_run.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_run.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8bc2923fa069f71d5bb00d4f272c235d6e7f0c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_run.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: run +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_run_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee4fabdc96a680e0482040f00daca794eaf87dfe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_sna_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_som.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_som.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88d7774c1b4c1c08367d8ebc79bc1ca7214cab9f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_som.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_som_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4e02aae282bdba08b8f057c71ab580a7ee9c031 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_swa_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72fa30ae7012379011b04a1fb3255fcad2a9a4e7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_tir.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_tir_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d98b3b681de265b99207db0d3d2347d952d05a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_xho.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_xho_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ef4eec0e46425b9b02771ee6a360f3498cc0a1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_yor_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_1/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews new file mode 100644 index 0000000000000000000000000000000000000000..c174d2c7ff991b749881260b1ccb93d63a5e9f26 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews @@ -0,0 +1,43 @@ +tag: +- masakhanews_tasks +- masakhanews_prompt_2 +- afrobench_TC_tasks +dataset_path: masakhane/masakhanews +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "business" + - "entertainment" + - "health" + - "politics" + - "religion" + - "sports" + - "technology" +should_decontaminate: true +doc_to_decontamination_query: headline_text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cee7619cfb4c215a9e05551b668d2ea4e9e517ca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_amh.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: 'Does this Amharic topic; ''{{headline_text}}'' belong to one of the + following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_amh_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3d6dd16c461eb7df5b730aa3283f6695f130503 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_eng.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: 'Does this English topic; ''{{headline_text}}'' belong to one of the + following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_eng_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c35a6a1d34753893185c2d0fcd5dd82e73853e35 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_fra.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: 'Does this French topic; ''{{headline_text}}'' belong to one of the following + categories: technology, business, politics, sports, health, entertainment, or religion? + category only + + + ' +include: masakhanews +task: masakhanews_fra_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93e9cc5a7f3e53fd8fc97ee1ecf8bfbb85911939 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_hau.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Does this Hausa topic; ''{{headline_text}}'' belong to one of the following + categories: technology, business, politics, sports, health, entertainment, or religion? + category only + + + ' +include: masakhanews +task: masakhanews_hau_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1638e435c76f6fdd671a6bf165d81debc4be3b3a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_ibo.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Does this Igbo topic; ''{{headline_text}}'' belong to one of the following + categories: technology, business, politics, sports, health, entertainment, or religion? + category only + + + ' +include: masakhanews +task: masakhanews_ibo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0010d0e1ad36356f8c5b9bccccffc62f46730d93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_lin.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: 'Does this Lingala topic; ''{{headline_text}}'' belong to one of the + following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_lin_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d526067289d1145ed494418408b0de55d51a74ca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_lug.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: 'Does this Luganda topic; ''{{headline_text}}'' belong to one of the + following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_lug_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd04c845d4eb1d46dd48052f0a582d7557e610a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_orm.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: 'Does this Afaan Oromoo topic; ''{{headline_text}}'' belong to one of + the following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_orm_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_pcm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_pcm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de685e3ac8a62a2e5ca5b3d0b16119a77caa1994 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_pcm.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: 'Does this Nigerian Pidgin topic; ''{{headline_text}}'' belong to one + of the following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_pcm_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_run.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_run.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62236d590bfc009043dd8dca6ab3c43edcaff995 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_run.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: run +doc_to_text: 'Does this Kirundi topic; ''{{headline_text}}'' belong to one of the + following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_run_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a97e176b865335804af5efe9e911894acbbcc78 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_sna.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: 'Does this Shona topic; ''{{headline_text}}'' belong to one of the following + categories: technology, business, politics, sports, health, entertainment, or religion? + category only + + + ' +include: masakhanews +task: masakhanews_sna_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_som.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_som.yaml new file mode 100644 index 0000000000000000000000000000000000000000..318b9b87beabeef3837c826cbf128b3b8d1b4e8d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_som.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: som +doc_to_text: 'Does this Somali topic; ''{{headline_text}}'' belong to one of the following + categories: technology, business, politics, sports, health, entertainment, or religion? + category only + + + ' +include: masakhanews +task: masakhanews_som_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75b9345f3229c6e105f4cd99e65a970c1061b7d4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_swa.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'Does this Swahili topic; ''{{headline_text}}'' belong to one of the + following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_swa_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..258a2bd3d7431083088e09f74044ce518cbaa7b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_tir.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: tir +doc_to_text: 'Does this Tigrinya topic; ''{{headline_text}}'' belong to one of the + following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_tir_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c4c3ac3abdb3b239406a2d36efc9331b1597d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_xho.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: 'Does this Xhosa topic; ''{{headline_text}}'' belong to one of the following + categories: technology, business, politics, sports, health, entertainment, or religion? + category only + + + ' +include: masakhanews +task: masakhanews_xho_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..067cf10632de430b650053d5537a733052e34b09 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_yor.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Does this Yoruba topic; ''{{headline_text}}'' belong to one of the following + categories: technology, business, politics, sports, health, entertainment, or religion? + category only + + + ' +include: masakhanews +task: masakhanews_yor_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_2/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews new file mode 100644 index 0000000000000000000000000000000000000000..ecc2108967078bb24a1efd15acdd8387d47e173c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews @@ -0,0 +1,43 @@ +tag: +- masakhanews_tasks +- masakhanews_prompt_3 +- afrobench_TC_tasks +dataset_path: masakhane/masakhanews +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "business" + - "entertainment" + - "health" + - "politics" + - "religion" + - "sports" + - "technology" +should_decontaminate: true +doc_to_decontamination_query: headline_text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dec10d2963dd00fce7e8dbd7f24f8a61a178e0a7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_amh.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Amharic statement below? Return only the category. \n\ + \ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_amh_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8b7159e215dd6bc5a766d51b06f77289e4ce1a7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_eng.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the English statement below? Return only the category. \n\ + \ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_eng_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..328316a8361d29a4db6ab882b46944fc65b2ff9b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_fra.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the French statement below? Return only the category. \n\n\ + text: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_fra_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c18ff2779cc9a9d149afe1eb7c438e3d18e8af2c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_hau.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Hausa statement below? Return only the category. \n\n\ + text: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_hau_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a91db840f2b72f041cffb827ea87520e28434cb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_ibo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Igbo statement below? Return only the category. \n\n\ + text: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_ibo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19c4cca2e5f65851b6c44a1baa6dd2842ce3bd5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_lin.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Lingala statement below? Return only the category. \n\ + \ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_lin_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e3d4319fc82762f63292da9f916166132e53a42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_lug.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Luganda statement below? Return only the category. \n\ + \ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_lug_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_orm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9bacf0420b81df236351a3698e37cf3eca8983e7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_orm.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Afaan Oromoo statement below? Return only the category.\ + \ \n\ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_orm_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_pcm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_pcm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e873becd56c21cd0d92841994a4fb6bed5119b51 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_pcm.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Nigerian Pidgin statement below? Return only the category.\ + \ \n\ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_pcm_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_run.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_run.yaml new file mode 100644 index 0000000000000000000000000000000000000000..307e13710dde72132a0df4011500aca4ccfd9e22 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_run.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: run +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Kirundi statement below? Return only the category. \n\ + \ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_run_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee69be3de11efe89bac3dd355f4a9ffe99206c37 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_sna.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Shona statement below? Return only the category. \n\n\ + text: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_sna_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_som.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_som.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c181fddb819a9f42f18a31141bc71f837f759cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_som.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: som +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Somali statement below? Return only the category. \n\n\ + text: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_som_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fbe1c4200f9953d603eae6262067facfc09fe694 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_swa.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Swahili statement below? Return only the category. \n\ + \ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_swa_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..110fc08778130c4880b904a682211df4800a2cd5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_xho.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Xhosa statement below? Return only the category. \n\n\ + text: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_xho_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d31e9b23fdfbdc5bbc85be20457c80c9b90c4a31 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_yor.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Yoruba statement below? Return only the category. \n\n\ + text: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_yor_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews new file mode 100644 index 0000000000000000000000000000000000000000..a1801f4e00b1d90a885ed9d73a14c2745cd73f01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews @@ -0,0 +1,43 @@ +tag: +- masakhanews_tasks +- masakhanews_prompt_4 +- afrobench_TC_tasks +dataset_path: masakhane/masakhanews +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "business" + - "entertainment" + - "health" + - "politics" + - "religion" + - "sports" + - "technology" +should_decontaminate: true +doc_to_decontamination_query: headline_text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a76305859c1e27a1b00b3be6492a76b207d313da --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_amh_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8567113756fde2e8e98c5f6f2f68073a5d14550b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_eng_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f86635f6d5edf28590264ae45a0f3546d868feb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_fra.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_fra_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1b7ce562b51486f41ce75c6716eda24d56caf1f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_hau_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d76a905d62d2e93f09608684592dff02c60f131c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_ibo_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0247529b5c4496cce3f52651f6969e894484bb1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_lin_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca02c0a5fcbd248c82e945646d932614c8e515e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_lug_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_pcm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_pcm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93ad9f482b3539efd0b4e7ab89b64ab75ce91147 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_pcm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_pcm_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_run.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_run.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5d985481f1b4443cadd4c6d1ef12424825e02cb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_run.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: run +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_run_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews new file mode 100644 index 0000000000000000000000000000000000000000..8d76af03ab044d68314853c0a0005a05141c1dca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews @@ -0,0 +1,43 @@ +tag: +- masakhanews_tasks +- masakhanews_prompt_5 +- afrobench_TC_tasks +dataset_path: masakhane/masakhanews +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "business" + - "entertainment" + - "health" + - "politics" + - "religion" + - "sports" + - "technology" +should_decontaminate: true +doc_to_decontamination_query: headline_text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..603d149d733336355b4874a5bbffe61786a9edd7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_fra.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "You are tasked with performing topic classification on the following\ + \ French text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_fra_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04a478cf6a5b63269c1ef2ef061d50fd08f95c11 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_hau.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Hausa text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_hau_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e22303fe79bca34cef9784b7ecea4fe1d1a39ab7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_lin.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Lingala text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_lin_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..310a7aeb5af2b998d57c6a793f27b00c8ab04029 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/utils.py @@ -0,0 +1,127 @@ +import argparse +import os + +import yaml + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_1": "Given the categories technology, business, politics, sports, health, entertainment, or religion; what category does the text: '{{headline}}' belong to: \n\n", + "prompt_2": f"Does this {lang} topic; " + "'{{headline}}' belong to one of the following categories: technology, business, politics, sports, health, entertainment, or religion? category only\n\n", + "prompt_3": f"You are an assistant able to classify topics in texts. \n\n" + f"Given the categories technology, religion, politics, sports, health, entertainment, or business; what is " + f"the topic of the {lang} statement below? Return only the category. " + "\n\ntext: {{headline}} \category:\n\n", + "prompt_4": "Label the following text as technology, religion, politics, sports, health, entertainment, or geography. Provide only the category as your " + "response. \n\ntext: {{headline}} \category: \n\n", + "prompt_5": f"You are tasked with performing topic classification on the following {lang} text. " + f"For each input, classify the topic as technology, business, politics, sports, health, entertainment, or religion. " + f"Use the following guidelines: \n\n " + f"technology: The text discusses scientific discoveries, technological advancements, or related topics. \n" + f"politics: The text covers political events, policies, or related topics. \n" + f"sports: The text talks about sports events, athletes, or related topics. \n" + f"health: The text addresses health issues, medical advancements, or related topics. \n" + f"entertainment: The text pertains to movies, music, celebrities, or related topics. \n" + f"religion: The text talks about relgions, religious institutions and beliefs or related topics. \n\n" + f"business: The text covers economy, business, or related topics. \n\n" + f"If the text contains multiple topics, choose the dominant topic. " + f"For ambiguous or unclear topics, select the category that best reflects the overall content. " + "Please provide a single classification for each input.\n\ntext: {{headline}} \category: \n\n", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "amh": "Amharic", + "eng": "English", + "fra": "French", + "hau": "Hausa", + "ibo": "Igbo", + "lin": "Lingala", + "lug": "Luganda", + "orm": "Afaan Oromoo", + "pcm": "Nigerian Pidgin", + "run": "Kirundi", + "sna": "Shona", + "som": "Somali", + "swa": "Swahili", + "tir": "Tigrinya", + "xho": "Xhosa", + "yor": "Yoruba", + } + + for lang in languages.keys(): + try: + file_name = f"masakhanews_{lang}.yaml" + task_name = f"masakhanews_{lang}_{mode}" + yaml_template = "masakhanews" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + "doc_to_text": prompt_func(mode, languages[lang]), + } + file_path = os.path.join(output_dir, mode) + os.makedirs(file_path, exist_ok=True) + + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + + PROMPT_CHOICES = ["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"] + parser.add_argument( + "--mode", + nargs="*", + default=PROMPT_CHOICES, + choices=PROMPT_CHOICES, + help="Prompt number(s)", + ) + args = parser.parse_args() + + for mode in args.mode: + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=mode) + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/masakhapos.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/masakhapos.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3fb1574eb32a0203198a4d210c788765cf476f34 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/masakhapos.yaml @@ -0,0 +1,13 @@ +group: masakhapos +task: + - masakhapos_prompt_1 + - masakhapos_prompt_2 + - masakhapos_prompt_3 + - masakhapos_prompt_4 + - masakhapos_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1c64e387ae638c83e30b1172f458c3976d20728 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_bam.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: bam +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_bam_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b08dacdef6912bf10bc3136726f28229eeb43d30 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_swa.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_swa_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_tsn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_tsn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbc20d6ea0ab4e613dc077eac01227c1d8ca198a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_tsn.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: tsn +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_tsn_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..11af3b877758759dc4d4eb34fbf8f99421d54f7b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_twi.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_twi_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_yaml new file mode 100644 index 0000000000000000000000000000000000000000..df148e8a8ab567d65dc12a36f60a0b3f753b8c86 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_yaml @@ -0,0 +1,32 @@ +tag: +- masakhapos_tasks +- masakhapos_prompt_5 +dataset_path: masakhane/masakhapos +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +generation_kwargs: + do_sample: false + until: + - + - <|im_end|> +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: !function utils.doc_to_target +should_decontaminate: true +doc_to_decontamination_query: "Sentence: {{token}}\nOutput:" +filter_list: + - filter: + - function: regex_pos + name: flexible-extract +metric_list: + - metric: acc + aggregation: !function utils.acc_score + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84bb266af31906017d066df113e7ca999579f744 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_yor.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_yor_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4ccc66d9cce30c1459494f0d5c21a71d1d3f58d4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_5/utils.py @@ -0,0 +1,55 @@ +from itertools import chain + +from sklearn.metrics import accuracy_score + +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + pos_tag_map = { + 0: "NOUN", + 1: "PUNCT", + 2: "ADP", + 3: "NUM", + 4: "SYM", + 5: "SCONJ", + 6: "ADJ", + 7: "PART", + 8: "DET", + 9: "CCONJ", + 10: "PROPN", + 11: "PRON", + 12: "X", + 13: "_", + 14: "ADV", + 15: "INTJ", + 16: "VERB", + 17: "AUX", + } + return [pos_tag_map[tag] for tag in doc["upos"]] + + +def acc_score(items): + unzipped_list = list(zip(*items)) + + golds, preds = unzipped_list[0], unzipped_list[1] + + # Flatten preds' inner lists + flattened_preds = [list(chain.from_iterable(p)) for p in preds] + + # Calculate the accuracy for each gold-pred pair + accuracy_scores = [] + for gold, pred in zip(golds, flattened_preds): + # Ensure both lists are of the same length, otherwise truncate to match + min_length = min(len(gold), len(pred)) + gold = gold[:min_length] + pred = pred[:min_length] + + # Calculate accuracy for the current pair and add to the list + accuracy = accuracy_score(gold, pred) + accuracy_scores.append(accuracy) + + mean_accuracy = ( + sum(accuracy_scores) / len(accuracy_scores) if accuracy_scores else 0 + ) + return mean_accuracy diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_pcm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_pcm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..536301688c02f9ca8ef4f576d9874ad624abe8fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_pcm.yaml @@ -0,0 +1,3 @@ +dataset_name: pcm +include: nollysenti +task: nollysenti_pcm_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti new file mode 100644 index 0000000000000000000000000000000000000000..76f664fee41316e4b8cf10faca4498c1e1c22916 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti @@ -0,0 +1,37 @@ +tag: + - afrobench_sentiment_tasks + - nollysenti_prompt_2 +dataset_path: Davlan/nollysenti +dataset_kwargs: {trust_remote_code: True} +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "positive" + - "negative" +should_decontaminate: true +doc_to_decontamination_query: review +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f87bce673c68bacdcf3e516bb58c116ada8209e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_hau.yaml @@ -0,0 +1,4 @@ +dataset_name: ha +include: nollysenti +doc_to_text: 'Does this Hausa movie description; "{{review}}" have a Positive or Negative sentiment? Labels only\n' +task: nollysenti_hau_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03c89d8bd05dec45bfc07f5af8c2dc8ed76388ae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_yor.yaml @@ -0,0 +1,4 @@ +dataset_name: yo +include: nollysenti +doc_to_text: 'Does this Yoruba movie description; "{{review}}" have a Positive or Negative sentiment? Labels only\n' +task: nollysenti_yor_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_2/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_2/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_2/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti new file mode 100644 index 0000000000000000000000000000000000000000..472928acdc7b964d60fbd0eb992af298319afcc4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti @@ -0,0 +1,37 @@ +tag: + - afrobench_sentiment_tasks + - nollysenti_prompt_3 +dataset_path: Davlan/nollysenti +dataset_kwargs: {trust_remote_code: True} +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "positive" + - "negative" +should_decontaminate: true +doc_to_decontamination_query: review +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df21a145c99fb1e7612868276e481724503460bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: en +doc_to_text: "You are an assistant able to detect sentiment in movie reviews. \n\nGiven\ + \ the sentiment labels Positive or Negative; what is the sentiment of the\ + \ English statement below? Return only the labels\n\nReview: {{review}}\n" +include: nollysenti +task: nollysenti_eng_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d15488d6e25022a68dae9874a3b77598fd22dc0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ha +doc_to_text: "You are an assistant able to detect sentiment in movie reviews. \n\nGiven\ + \ the sentiment labels Positive or Negative; what is the sentiment of the\ + \ Hausa statement below? Return only the labels\n\nReview: {{review}}\n" +include: nollysenti +task: nollysenti_hau_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f6bb7b29581858b860b5919afbab5e5b22ebc28 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ig +doc_to_text: "You are an assistant able to detect sentiment in movie reviews. \n\nGiven\ + \ the sentiment labels Positive or Negative; what is the sentiment of the\ + \ Igbo statement below? Return only the labels\n\nReview: {{review}}\n" +include: nollysenti +task: nollysenti_ibo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_pcm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_pcm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f98519f3ed329da73ab2272fd33305670d8f2ec1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_pcm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "You are an assistant able to detect sentiment in movie reviews. \n\nGiven\ + \ the sentiment labels Positive or Negative; what is the sentiment of the\ + \ Naija Pidgin statement below? Return only the labels\n\nReview: {{review}}\n" +include: nollysenti +task: nollysenti_pcm_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd64d1eda4fa7048690527046e71c6af21eb0d51 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yo +doc_to_text: "You are an assistant able to detect sentiment in movie reviews. \n\nGiven\ + \ the sentiment labels Positive or Negative; what is the sentiment of the\ + \ Yoruba statement below? Return only the labels\n\nReview: {{review}}\n" +include: nollysenti +task: nollysenti_yor_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_3/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti new file mode 100644 index 0000000000000000000000000000000000000000..de1bb486dc1c84ea828d1cb99deb16af6e3f1644 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti @@ -0,0 +1,37 @@ +tag: + - afrobench_sentiment_tasks + - nollysenti_prompt_4 +dataset_path: Davlan/nollysenti +dataset_kwargs: {trust_remote_code: True} +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "positive" + - "negative" +should_decontaminate: true +doc_to_decontamination_query: review +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8e01ab6efb4450b392b7d6278088c7f74114f61 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: en +doc_to_text: "Label the following text as Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_eng_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..abc9570484fbd79acebb9ba2b7be840bb9391c4d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_hau.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ha +doc_to_text: "Label the following text as Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_hau_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8962cf729075203d9c853470791aa15f7eb97023 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ig +doc_to_text: "Label the following text as Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_ibo_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_pcm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_pcm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36d43b795461972411b56413b1bc11386cc34d78 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_pcm.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "Label the following text as Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_pcm_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c100c4dd367e2d610e8881d0d7d932c3473f38c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_yor.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yo +doc_to_text: "Label the following text as Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_yor_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_4/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti new file mode 100644 index 0000000000000000000000000000000000000000..2e25f2f088edcb81f754f3b7fd7f9a5e92e18b12 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti @@ -0,0 +1,37 @@ +tag: + - afrobench_sentiment_tasks + - nollysenti_prompt_5 +dataset_path: Davlan/nollysenti +dataset_kwargs: {trust_remote_code: True} +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "positive" + - "negative" +should_decontaminate: true +doc_to_decontamination_query: review +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d485ffe154c61f91924a5c0015e5defeb8ea83a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_eng.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: en +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ English text. For each input, classify the sentiment as positive, negative.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_eng_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ed16af77a33c39aa1569a38047ef92091837152 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_hau.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ha +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Hausa text. For each input, classify the sentiment as positive, negative.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_hau_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c75f26900298951c5934b17964ca0cd744d86726 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_ibo.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ig +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Igbo text. For each input, classify the sentiment as positive, negative.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_ibo_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_pcm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_pcm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29b5cda0b66b083a2cbcdf8d6750d447e7890519 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_pcm.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Naija Pidgin text. For each input, classify the sentiment as positive, negative.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_pcm_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1aea0284e191356e15db16036a4d1abfbc1c5aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_yor.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: yo +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Yoruba text. For each input, classify the sentiment as positive, negative.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_yor_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/nollysenti/prompt_5/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/README.md b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d68cf8c99cb4d7cb8c68eb7d015e6cb26daca3cb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/README.md @@ -0,0 +1,38 @@ +# + +## Paper +Title: `NTREX-128 – News Test References for MT Evaluation of 128 Languages` + +Paper Link: https://aclanthology.org/2022.sumeval-1.4/ + +## Abstract +>We release NTREX-128, a data set for machine translation (MT) evaluation from English into a total of 128 target languages. The paper describes the data creation process and proposes a quality filtering method based on human evaluation. We show experimental results which confirm that the directionality of test sets translation indeed plays an important role wrt. the usefulness of the corresponding metrics’ scores. Thus, we recommend that the NTREX-128 data set should be used for evaluation of Englishsourced translation models but not in reverse direction. The test set release introduces another benchmark for the evaluation of massively multilingual machine translation research. + +HomePage: https://github.com/MicrosoftTranslator/NTREX + +### Citation + +``` +@inproceedings{federmann-etal-2022-ntrex, + title = "{NTREX}-128 {--} News Test References for {MT} Evaluation of 128 Languages", + author = "Federmann, Christian and + Kocmi, Tom and + Xin, Ying", + editor = "Ahuja, Kabir and + Anastasopoulos, Antonios and + Patra, Barun and + Neubig, Graham and + Choudhury, Monojit and + Dandapat, Sandipan and + Sitaram, Sunayana and + Chaudhary, Vishrav", + booktitle = "Proceedings of the First Workshop on Scaling Up Multilingual Evaluation", + month = nov, + year = "2022", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2022.sumeval-1.4/", + doi = "10.18653/v1/2022.sumeval-1.4", + pages = "21--24" +} +``` diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/gen_utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/gen_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ba549de25b69b0892f6e80c923c44f7ca001cd79 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/gen_utils.py @@ -0,0 +1,171 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang, lang_dict): + language_column_name = f"sentence_{lang}" + prompt_map = { + "prompt_1": f"{lang_dict[lang]}: {{{{{language_column_name}}}}} \nEnglish: ", + "prompt_1_reverse": f"English: {{{{sentence_eng_Latn}}}} \n{lang_dict[lang]}: ", + "prompt_2": f"You are a translation expert. Translate the following {lang_dict[lang]} sentences to English \n" + f"{lang_dict[lang]}: {{{{{language_column_name}}}}}\nEnglish: ", + "prompt_2_reverse": f"You are a translation expert. Translate the following English sentences to " + f"{lang_dict[lang]} " + "\nEnglish: {{sentence_eng_Latn}} " + f"\n{lang_dict[lang]}: ", + "prompt_3": f"As a {lang_dict[lang]} and English linguist, translate the following {lang_dict[lang]} sentences " + f"to English \n{lang_dict[lang]}: {{{{{language_column_name}}}}}\nEnglish: ", + "prompt_3_reverse": f"As a {lang_dict[lang]} and English linguist, translate the following English sentences to " + f"{lang_dict[lang]} " + "\nEnglish: {{sentence_eng_Latn}} " + f"\n{lang_dict[lang]}: ", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str, reverse: bool) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "afr_Latn": "Afrikaans", + "amh_Ethi": "Amharic", + "arb_Arab": "Arabic", + "bem_Latn": "Bemba", + "ewe_Latn": "Ewe", + "fra_Latn": "French", + "hau_Latn": "Hausa", + "ibo_Latn": "Igbo", + "kin_Latn": "Kinyarwanda", + "mey_Arab": "Hassaniya Arabic", + "mlg_Latn": "Malagasy", + "msa_Latn": "Malay", + "nde_Latn": "North Ndebele", + "nso_Latn": "Northern Sotho", + "nya_Latn": "Chichewa", + "orm_Ethi": "Oromo", + "shi_Arab": "Tachelhit", + "sna_Latn": "Shona (Latin)", + "som_Latn": "Somali", + "ssw_Latn": "Swati", + "swa_Latn": "Swahili", + "tam_Taml": "Tamil", + "tel_Telu": "Telugu", + "tir_Ethi": "Tigrinya", + "ton_Latn": "Tongan", + "tsn_Latn": "Tswana", + "urd_Arab": "Urdu", + "ven_Latn": "Venda", + "wol_Latn": "Wolof", + "xho_Latn": "Xhosa", + "yor_Latn": "Yoruba", + "zul_Latn": "Zulu", + } + + for lang in languages.keys(): + try: + if not reverse: + file_name = f"ntrex_{lang}-eng_Latn.yaml" + task_name = f"ntrex_{lang}-eng_Latn_{mode}" + yaml_template = "ntrex" + yaml_details = { + "include": yaml_template, + "dataset_name": f"{lang}", + "task": task_name, + "doc_to_target": "sentence_eng_Latn", + "doc_to_text": prompt_func(mode, lang, languages), + } + os.makedirs(f"{output_dir}/{mode}/african-english", exist_ok=True) + with open( + f"{output_dir}/{mode}/african-english/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + else: + file_name = f"ntrex_eng_Latn-{lang}.yaml" + task_name = f"ntrex_eng_Latn-{lang}_{mode}" + yaml_template = "ntrex" + yaml_details = { + "include": yaml_template, + "dataset_name": f"{lang}", + "task": task_name, + "doc_to_target": f"sentence_{lang}", + "doc_to_text": prompt_func(f"{mode}_reverse", lang, languages), + } + os.makedirs(f"{output_dir}/{mode}/english-african", exist_ok=True) + with open( + f"{output_dir}/{mode}/english-african/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_1", + choices=["prompt_1", "prompt_2", "prompt_3"], + help="Prompt number", + ) + parser.add_argument( + "--reverse", + default=False, + choices=[True, False], + help="Reverse the translation direction", + ) + args = parser.parse_args() + + gen_lang_yamls( + output_dir=args.output_dir, + overwrite=args.overwrite, + mode=args.mode, + reverse=args.reverse, + ) + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/ntrex.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/ntrex.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c30b08cea2ffdbf775cfeeb8957c47e9e807518a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/ntrex.yaml @@ -0,0 +1,14 @@ +group: african_ntrex +task: + - ntrex_eng-afr_prompt_1 + - ntrex_eng-afr_prompt_2 + - ntrex_eng-afr_prompt_3 + - ntrex_afr-eng_prompt_1 + - ntrex_afr-eng_prompt_2 + - ntrex_afr-eng_prompt_3 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex new file mode 100644 index 0000000000000000000000000000000000000000..3c2659d752c9f14412d23f3c1e553fbb03a16b03 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex @@ -0,0 +1,26 @@ +tag: +- ntrex_tasks +- ntrex_afr-eng +- ntrex_afr-eng_prompt_1 +- afrobench_MT_tasks +dataset_path: masakhane/ntrex_african +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: test +fewshot_split: test +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_afr_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_afr_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb11904366801d649186548e124027489497a4cb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_afr_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Afrikaans: {{sentence_afr_Latn}} \nEnglish: " +include: ntrex +task: ntrex_afr_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_amh_Ethi-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_amh_Ethi-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0114a212b89bee62243b3adedad49066998d1785 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_amh_Ethi-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "Amharic: {{sentence_amh_Ethi}} \nEnglish: " +include: ntrex +task: ntrex_amh_Ethi-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_arb_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_arb_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ddc8c4bbd403a3b83c15172d119ae183247c522 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_arb_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: arb_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "Arabic: {{sentence_arb_Arab}} \nEnglish: " +include: ntrex +task: ntrex_arb_Arab-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_bem_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_bem_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c33ab35a18175300ffbf938b2431652ecf86017e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_bem_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Bemba: {{sentence_bem_Latn}} \nEnglish: " +include: ntrex +task: ntrex_bem_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ewe_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ewe_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5f69c0051ac2292ef1282ac6c8844ee61bc5148 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ewe_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Ewe: {{sentence_ewe_Latn}} \nEnglish: " +include: ntrex +task: ntrex_ewe_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_fra_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_fra_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa3fad61684684f7155bf40704397cff7d5bcbc8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_fra_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "French: {{sentence_fra_Latn}} \nEnglish: " +include: ntrex +task: ntrex_fra_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_hau_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_hau_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b6d0f28b84d4c89d96f3db9de8478201265fade --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_hau_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Hausa: {{sentence_hau_Latn}} \nEnglish: " +include: ntrex +task: ntrex_hau_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ibo_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ibo_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..992598614c1d9fb0929ca024260a31b953a1204e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ibo_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Igbo: {{sentence_ibo_Latn}} \nEnglish: " +include: ntrex +task: ntrex_ibo_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_kin_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_kin_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eee96a62b961371c1fd1f069e97cd94ebef5b4d5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_kin_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Kinyarwanda: {{sentence_kin_Latn}} \nEnglish: " +include: ntrex +task: ntrex_kin_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_mey_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_mey_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6650e644ad9b84df3c93bb6622543f8984bc4f8b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_mey_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: mey_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "Hassaniya Arabic: {{sentence_mey_Arab}} \nEnglish: " +include: ntrex +task: ntrex_mey_Arab-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_mlg_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_mlg_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..375522c5c8560747a2775ec380b4964296dec7e3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_mlg_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: mlg_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Malagasy: {{sentence_mlg_Latn}} \nEnglish: " +include: ntrex +task: ntrex_mlg_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_msa_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_msa_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65aaaa8014abf84963112a1b7f0239f4129c20bb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_msa_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: msa_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Malay: {{sentence_msa_Latn}} \nEnglish: " +include: ntrex +task: ntrex_msa_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nde_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nde_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d63548fb439470b4d46fb7225fa521f31becc77f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nde_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: nde_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "North Ndebele: {{sentence_nde_Latn}} \nEnglish: " +include: ntrex +task: ntrex_nde_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nso_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nso_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4cf1cccf8a2562b0c958457561c7c4c9a5ae6776 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nso_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Northern Sotho: {{sentence_nso_Latn}} \nEnglish: " +include: ntrex +task: ntrex_nso_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nya_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nya_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee4ac6d73f198367a96c684921e6e65e9a0adea7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nya_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Chichewa: {{sentence_nya_Latn}} \nEnglish: " +include: ntrex +task: ntrex_nya_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_orm_Ethi-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_orm_Ethi-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..446873065b536f58bfa12e5886f49edd1b7ea5ee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_orm_Ethi-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: orm_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "Oromo: {{sentence_orm_Ethi}} \nEnglish: " +include: ntrex +task: ntrex_orm_Ethi-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_shi_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_shi_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10972893f3f453f91d12845d9fea3e43558c1fc4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_shi_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: shi_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "Tachelhit: {{sentence_shi_Arab}} \nEnglish: " +include: ntrex +task: ntrex_shi_Arab-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_sna_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_sna_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63d83528835e8ae79f82d09007a4494ccaf1229c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_sna_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Shona (Latin): {{sentence_sna_Latn}} \nEnglish: " +include: ntrex +task: ntrex_sna_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_som_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_som_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6eb91e0310fcebd6483a3d43aca793e3a6934b9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_som_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Somali: {{sentence_som_Latn}} \nEnglish: " +include: ntrex +task: ntrex_som_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ssw_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ssw_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48c5c10973911aa3b779071ffa96513e1e1f7a7a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ssw_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Swati: {{sentence_ssw_Latn}} \nEnglish: " +include: ntrex +task: ntrex_ssw_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_swa_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_swa_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..863222f7325fab67ff5afe3a13bef0cc0f4df035 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_swa_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: swa_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Swahili: {{sentence_swa_Latn}} \nEnglish: " +include: ntrex +task: ntrex_swa_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tam_Taml-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tam_Taml-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..993b480f20e34eab5f1c4cdfb644e09e0e978264 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tam_Taml-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tam_Taml +doc_to_target: sentence_eng_Latn +doc_to_text: "Tamil: {{sentence_tam_Taml}} \nEnglish: " +include: ntrex +task: ntrex_tam_Taml-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tel_Telu-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tel_Telu-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d91e9a1f762a013ed992d04a2c9e9f0049d8f7eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tel_Telu-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tel_Telu +doc_to_target: sentence_eng_Latn +doc_to_text: "Telugu: {{sentence_tel_Telu}} \nEnglish: " +include: ntrex +task: ntrex_tel_Telu-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tir_Ethi-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tir_Ethi-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f23f332c1ca392e44c62638d8e39a79f8839b54d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tir_Ethi-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "Tigrinya: {{sentence_tir_Ethi}} \nEnglish: " +include: ntrex +task: ntrex_tir_Ethi-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ton_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ton_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5676a1a99997aca3d0bfc4120003ccb4edef3099 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ton_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ton_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Tongan: {{sentence_ton_Latn}} \nEnglish: " +include: ntrex +task: ntrex_ton_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tsn_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tsn_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..caa0f9e57b1d93a4c074cca1c816ded7a93c3eb6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tsn_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Tswana: {{sentence_tsn_Latn}} \nEnglish: " +include: ntrex +task: ntrex_tsn_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_urd_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_urd_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e07e6787868ec0a56e7b79b2246fcd2211c19d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_urd_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: urd_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "Urdu: {{sentence_urd_Arab}} \nEnglish: " +include: ntrex +task: ntrex_urd_Arab-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ven_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ven_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ba8ceaf4921b087cf38dce53a8c9bb49c359389 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ven_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ven_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Venda: {{sentence_ven_Latn}} \nEnglish: " +include: ntrex +task: ntrex_ven_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_wol_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_wol_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dcacb69de3f8fd83c5714494665cfb7f8cc7be1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_wol_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Wolof: {{sentence_wol_Latn}} \nEnglish: " +include: ntrex +task: ntrex_wol_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_xho_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_xho_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b6abc9dcbf53879148418592fd155f95026bba8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_xho_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Xhosa: {{sentence_xho_Latn}} \nEnglish: " +include: ntrex +task: ntrex_xho_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_yor_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_yor_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e98aecd5b188aabf46c2c00b9a126616fee55f6f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_yor_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Yoruba: {{sentence_yor_Latn}} \nEnglish: " +include: ntrex +task: ntrex_yor_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_zul_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_zul_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a38abee1148ad1b77a5395afa48621070ad3c239 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_zul_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Zulu: {{sentence_zul_Latn}} \nEnglish: " +include: ntrex +task: ntrex_zul_Latn-eng_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex new file mode 100644 index 0000000000000000000000000000000000000000..2b5aa84f990e10804a9cdc8ca69901bfb55e5d71 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex @@ -0,0 +1,26 @@ +tag: +- ntrex_tasks +- ntrex_eng-afr +- ntrex_eng-afr_prompt_1 +- afrobench_MT_tasks +dataset_path: masakhane/ntrex_african +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: test +fewshot_split: test +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-afr_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-afr_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40471f80151bacf355f8bf8ff617027f9da68ef7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-afr_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_target: sentence_afr_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nAfrikaans: " +include: ntrex +task: ntrex_eng_Latn-afr_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-amh_Ethi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-amh_Ethi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e4dfba5dc799649532e9e6b28c862b25afb9566 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-amh_Ethi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_target: sentence_amh_Ethi +doc_to_text: "English: {{sentence_eng_Latn}} \nAmharic: " +include: ntrex +task: ntrex_eng_Latn-amh_Ethi_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-arb_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-arb_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a248a9ac6da1668ce1fab555fb7ad586cf0acaa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-arb_Arab.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: arb_Arab +doc_to_target: sentence_arb_Arab +doc_to_text: "English: {{sentence_eng_Latn}} \nArabic: " +include: ntrex +task: ntrex_eng_Latn-arb_Arab_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-bem_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-bem_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..035c682256b81ca9cc7dda1aebfc9ac130a75762 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-bem_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_target: sentence_bem_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nBemba: " +include: ntrex +task: ntrex_eng_Latn-bem_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ewe_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ewe_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5deae5c56b3bb203b372298207e7fa8d79cfb58 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ewe_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_target: sentence_ewe_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nEwe: " +include: ntrex +task: ntrex_eng_Latn-ewe_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-fra_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-fra_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf079cf440f75a35edbea04e8afa0703ab0eea7b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-fra_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_target: sentence_fra_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nFrench: " +include: ntrex +task: ntrex_eng_Latn-fra_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-hau_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-hau_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..377acbfb8ef84be01d8657907a33d1f141b66795 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-hau_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_target: sentence_hau_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nHausa: " +include: ntrex +task: ntrex_eng_Latn-hau_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ibo_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ibo_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c3a14dfa2200c29eb83825a6efb202905e6e78f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ibo_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_target: sentence_ibo_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nIgbo: " +include: ntrex +task: ntrex_eng_Latn-ibo_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-kin_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-kin_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec14399e37649d7671f81f5348d74e76235df4e3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-kin_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_target: sentence_kin_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nKinyarwanda: " +include: ntrex +task: ntrex_eng_Latn-kin_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-mey_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-mey_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb696cc5ac25f1f43c276c34e26b97b7c82efaee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-mey_Arab.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: mey_Arab +doc_to_target: sentence_mey_Arab +doc_to_text: "English: {{sentence_eng_Latn}} \nHassaniya Arabic: " +include: ntrex +task: ntrex_eng_Latn-mey_Arab_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-mlg_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-mlg_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..035c98c373ff6738310cb280cd617df60c8b6a2a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-mlg_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: mlg_Latn +doc_to_target: sentence_mlg_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nMalagasy: " +include: ntrex +task: ntrex_eng_Latn-mlg_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-msa_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-msa_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4c6b7d7f1f904ce5fe6061eb7c4c8caef86a8af --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-msa_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: msa_Latn +doc_to_target: sentence_msa_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nMalay: " +include: ntrex +task: ntrex_eng_Latn-msa_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nde_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nde_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c66b44beee186f47ea9f8b4d62776d60e4be3ba9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nde_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: nde_Latn +doc_to_target: sentence_nde_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nNorth Ndebele: " +include: ntrex +task: ntrex_eng_Latn-nde_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nso_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nso_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab6cf296c3959910f99b2edd6354d49259da7ce4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nso_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_target: sentence_nso_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nNorthern Sotho: " +include: ntrex +task: ntrex_eng_Latn-nso_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nya_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nya_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74cbd1ffed9675feaff5ead68f147fc2572b4edd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nya_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_target: sentence_nya_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nChichewa: " +include: ntrex +task: ntrex_eng_Latn-nya_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-orm_Ethi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-orm_Ethi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad875cab5b7012caecd06b99a8d7047ad50c403c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-orm_Ethi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: orm_Ethi +doc_to_target: sentence_orm_Ethi +doc_to_text: "English: {{sentence_eng_Latn}} \nOromo: " +include: ntrex +task: ntrex_eng_Latn-orm_Ethi_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-shi_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-shi_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5441bbdb6ea535f01c71753b9df5ee3290a7cac3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-shi_Arab.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: shi_Arab +doc_to_target: sentence_shi_Arab +doc_to_text: "English: {{sentence_eng_Latn}} \nTachelhit: " +include: ntrex +task: ntrex_eng_Latn-shi_Arab_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-sna_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-sna_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0bed0f6c195e7945329b7d26b50bb5d2abd62c90 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-sna_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_target: sentence_sna_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nShona (Latin): " +include: ntrex +task: ntrex_eng_Latn-sna_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-som_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-som_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e4aafdfc79bd2e31747847ec081ae15f3799dc3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-som_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_target: sentence_som_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSomali: " +include: ntrex +task: ntrex_eng_Latn-som_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ssw_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ssw_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa18ebf233e0cdbfd5b7d692356f0eacc1cf669a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ssw_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_target: sentence_ssw_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSwati: " +include: ntrex +task: ntrex_eng_Latn-ssw_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-swa_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-swa_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7079ec01354ca1d56fa593c4b2a5dab668f5c0c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-swa_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: swa_Latn +doc_to_target: sentence_swa_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSwahili: " +include: ntrex +task: ntrex_eng_Latn-swa_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tam_Taml.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tam_Taml.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7e42a36beee8d83d057b6daf7b6cfa488b2d90f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tam_Taml.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tam_Taml +doc_to_target: sentence_tam_Taml +doc_to_text: "English: {{sentence_eng_Latn}} \nTamil: " +include: ntrex +task: ntrex_eng_Latn-tam_Taml_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tel_Telu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tel_Telu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db8eb6b20ef17fb518b1c45a8753e72f205a7e41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tel_Telu.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tel_Telu +doc_to_target: sentence_tel_Telu +doc_to_text: "English: {{sentence_eng_Latn}} \nTelugu: " +include: ntrex +task: ntrex_eng_Latn-tel_Telu_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tir_Ethi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tir_Ethi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45c6ae84c642d58db1ebdbf45feb112c4e872bea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tir_Ethi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_target: sentence_tir_Ethi +doc_to_text: "English: {{sentence_eng_Latn}} \nTigrinya: " +include: ntrex +task: ntrex_eng_Latn-tir_Ethi_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ton_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ton_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a680a2c67f5226248043a9d8325b94f7fa4ad57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ton_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ton_Latn +doc_to_target: sentence_ton_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nTongan: " +include: ntrex +task: ntrex_eng_Latn-ton_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tsn_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tsn_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5a7a4ca261a1b8bfcdd1614eaa167c81c46c1d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tsn_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_target: sentence_tsn_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nTswana: " +include: ntrex +task: ntrex_eng_Latn-tsn_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-urd_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-urd_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ee69ded9fac3efbc400bbf39aadb529eee26e3d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-urd_Arab.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: urd_Arab +doc_to_target: sentence_urd_Arab +doc_to_text: "English: {{sentence_eng_Latn}} \nUrdu: " +include: ntrex +task: ntrex_eng_Latn-urd_Arab_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ven_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ven_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4277ce08a5d44f22996d704e0bfbd7461103a0ab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ven_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ven_Latn +doc_to_target: sentence_ven_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nVenda: " +include: ntrex +task: ntrex_eng_Latn-ven_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-wol_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-wol_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dea533ee5e959705c664d5b6e2ee10244c81d3f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-wol_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_target: sentence_wol_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nWolof: " +include: ntrex +task: ntrex_eng_Latn-wol_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-xho_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-xho_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62ab64bfd5e1a6d7a92e1491824047c4853b7e56 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-xho_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_target: sentence_xho_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nXhosa: " +include: ntrex +task: ntrex_eng_Latn-xho_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-yor_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-yor_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d96624ae3b9faeacd9b13bf8dcbaf95dafd1040 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-yor_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_target: sentence_yor_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nYoruba: " +include: ntrex +task: ntrex_eng_Latn-yor_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-zul_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-zul_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db60fb59821685f837f5f184647564f3e18f4927 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-zul_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_target: sentence_zul_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nZulu: " +include: ntrex +task: ntrex_eng_Latn-zul_Latn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex new file mode 100644 index 0000000000000000000000000000000000000000..3dc29226bf4677ee34836dbc0c5c206cbb1744bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex @@ -0,0 +1,25 @@ +tag: +- ntrex_afr-eng +- ntrex_afr-eng_prompt_2 +- afrobench_MT_tasks +dataset_path: masakhane/ntrex_african +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: test +fewshot_split: test +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_afr_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_afr_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16cfc7d5d0811aec8fca3bcbc7a436f74391cda5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_afr_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Afrikaans sentences\ + \ to English \nAfrikaans: {{sentence_afr_Latn}}\nEnglish: " +include: ntrex +task: ntrex_afr_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_amh_Ethi-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_amh_Ethi-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20e88c366d9c477928abda6bebd2a73d26d00e36 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_amh_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Amharic sentences\ + \ to English \nAmharic: {{sentence_amh_Ethi}}\nEnglish: " +include: ntrex +task: ntrex_amh_Ethi-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_arb_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_arb_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a88a478a12a99d5910360dab8b6fa6fac1b78601 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_arb_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arb_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Arabic sentences\ + \ to English \nArabic: {{sentence_arb_Arab}}\nEnglish: " +include: ntrex +task: ntrex_arb_Arab-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_bem_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_bem_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e114a3464d6cb98baf2374ccaacbc45c3f91240 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_bem_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Bemba sentences\ + \ to English \nBemba: {{sentence_bem_Latn}}\nEnglish: " +include: ntrex +task: ntrex_bem_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ewe_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ewe_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e4facd5106291d0fe52d5315d1f6a88a6f32afe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ewe_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Ewe sentences\ + \ to English \nEwe: {{sentence_ewe_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ewe_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_fra_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_fra_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad46aedf727a431a166cac1b9ec45be707feb9bb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_fra_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following French sentences\ + \ to English \nFrench: {{sentence_fra_Latn}}\nEnglish: " +include: ntrex +task: ntrex_fra_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_hau_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_hau_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..018a63963e8aeedeb3457a49cbf3d97adf4e8c82 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_hau_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Hausa sentences\ + \ to English \nHausa: {{sentence_hau_Latn}}\nEnglish: " +include: ntrex +task: ntrex_hau_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ibo_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ibo_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b93d2d863d60ded18b4e746badafe81e9a3e917 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ibo_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Igbo sentences\ + \ to English \nIgbo: {{sentence_ibo_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ibo_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_kin_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_kin_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45b18a640b749e848a8d7df9c01ac2121afb5c2d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_kin_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kinyarwanda sentences\ + \ to English \nKinyarwanda: {{sentence_kin_Latn}}\nEnglish: " +include: ntrex +task: ntrex_kin_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_mey_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_mey_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d155b62c828b30e1505e194d3a93960ed707c1aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_mey_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mey_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Hassaniya Arabic\ + \ sentences to English \nHassaniya Arabic: {{sentence_mey_Arab}}\nEnglish: " +include: ntrex +task: ntrex_mey_Arab-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_mlg_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_mlg_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10a7507bae076af1c5aec92ec0db65da9b94f876 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_mlg_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mlg_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Malagasy sentences\ + \ to English \nMalagasy: {{sentence_mlg_Latn}}\nEnglish: " +include: ntrex +task: ntrex_mlg_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_msa_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_msa_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be65a0ff07f372df2e3027373aebd4e0176e14ee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_msa_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: msa_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Malay sentences\ + \ to English \nMalay: {{sentence_msa_Latn}}\nEnglish: " +include: ntrex +task: ntrex_msa_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nde_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nde_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4a39fc2c31bc63eb27fdbfb78edaa8c8c59e0ee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nde_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nde_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following North Ndebele\ + \ sentences to English \nNorth Ndebele: {{sentence_nde_Latn}}\nEnglish: " +include: ntrex +task: ntrex_nde_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nso_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nso_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..290122fab7df120e79d478e81d3cc39cc60e61fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nso_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Northern Sotho\ + \ sentences to English \nNorthern Sotho: {{sentence_nso_Latn}}\nEnglish: " +include: ntrex +task: ntrex_nso_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nya_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nya_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de365e011b86d650b6defb5a6fd7abb4a7a0feef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nya_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Chichewa sentences\ + \ to English \nChichewa: {{sentence_nya_Latn}}\nEnglish: " +include: ntrex +task: ntrex_nya_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_orm_Ethi-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_orm_Ethi-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebe353d1fc9de1f7f89e4f783a57afe0a3699e7f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_orm_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Oromo sentences\ + \ to English \nOromo: {{sentence_orm_Ethi}}\nEnglish: " +include: ntrex +task: ntrex_orm_Ethi-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_shi_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_shi_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2db11ae54e39b0e8a5c5669489fdcd5a81bce29 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_shi_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: shi_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tachelhit sentences\ + \ to English \nTachelhit: {{sentence_shi_Arab}}\nEnglish: " +include: ntrex +task: ntrex_shi_Arab-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_sna_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_sna_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25600d6347d1973e1a3c4c8f236093044f2f83fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_sna_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Shona (Latin)\ + \ sentences to English \nShona (Latin): {{sentence_sna_Latn}}\nEnglish: " +include: ntrex +task: ntrex_sna_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_som_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_som_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ea6a71d9a423fad3ff285bbc53b3d7f440fac57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_som_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Somali sentences\ + \ to English \nSomali: {{sentence_som_Latn}}\nEnglish: " +include: ntrex +task: ntrex_som_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ssw_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ssw_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2e690a6a49b0e355df71f413522e6905f7601d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ssw_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Swati sentences\ + \ to English \nSwati: {{sentence_ssw_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ssw_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_swa_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_swa_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e609435f8482f5e2f4daa5253fef21dedcf36a3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_swa_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Swahili sentences\ + \ to English \nSwahili: {{sentence_swa_Latn}}\nEnglish: " +include: ntrex +task: ntrex_swa_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tam_Taml-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tam_Taml-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2c9f278f1353e66e341329e7fc9686169ab309e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tam_Taml-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tam_Taml +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tamil sentences\ + \ to English \nTamil: {{sentence_tam_Taml}}\nEnglish: " +include: ntrex +task: ntrex_tam_Taml-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tel_Telu-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tel_Telu-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15dc359805fb8570302d31ac432f5fd557cca2b9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tel_Telu-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tel_Telu +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Telugu sentences\ + \ to English \nTelugu: {{sentence_tel_Telu}}\nEnglish: " +include: ntrex +task: ntrex_tel_Telu-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tir_Ethi-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tir_Ethi-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f0bb2b835f9ad4e577b7e415ab7cfea484c1f44 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tir_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tigrinya sentences\ + \ to English \nTigrinya: {{sentence_tir_Ethi}}\nEnglish: " +include: ntrex +task: ntrex_tir_Ethi-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ton_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ton_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84f7d281eea533cb634ea958922ca7041a6e24f8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ton_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ton_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tongan sentences\ + \ to English \nTongan: {{sentence_ton_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ton_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tsn_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tsn_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a56996418d193d01df084b07b144f240ec45e7b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tsn_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tswana sentences\ + \ to English \nTswana: {{sentence_tsn_Latn}}\nEnglish: " +include: ntrex +task: ntrex_tsn_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_urd_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_urd_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47a47875969c6bb7d8032570beee3dcc4303b734 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_urd_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: urd_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Urdu sentences\ + \ to English \nUrdu: {{sentence_urd_Arab}}\nEnglish: " +include: ntrex +task: ntrex_urd_Arab-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ven_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ven_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f27b185d8d5c2c2fc0e7b4eb273f5fd8c2241ca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ven_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ven_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Venda sentences\ + \ to English \nVenda: {{sentence_ven_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ven_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_wol_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_wol_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa2da55ce1e8cc1b76e87dda701ae9e12e2976b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_wol_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Wolof sentences\ + \ to English \nWolof: {{sentence_wol_Latn}}\nEnglish: " +include: ntrex +task: ntrex_wol_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_xho_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_xho_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b504cd3126d1a422f94c10b7677c7bd92f0d9311 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_xho_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Xhosa sentences\ + \ to English \nXhosa: {{sentence_xho_Latn}}\nEnglish: " +include: ntrex +task: ntrex_xho_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_yor_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_yor_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03c4cbacd791035cd1757d5ca0ed14b546b445e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_yor_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Yoruba sentences\ + \ to English \nYoruba: {{sentence_yor_Latn}}\nEnglish: " +include: ntrex +task: ntrex_yor_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_zul_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_zul_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..760abb6f0253c993c37413c88c8dfced632cdd84 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_zul_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Zulu sentences\ + \ to English \nZulu: {{sentence_zul_Latn}}\nEnglish: " +include: ntrex +task: ntrex_zul_Latn-eng_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex new file mode 100644 index 0000000000000000000000000000000000000000..8dd411c3b78988b12ea421df33cf6aaa6caee91c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex @@ -0,0 +1,25 @@ +tag: +- ntrex_eng-afr +- ntrex_eng-afr_prompt_2 +- afrobench_MT_tasks +dataset_path: masakhane/ntrex_african +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: test +fewshot_split: test +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-afr_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-afr_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..678e5b21721a6da7f67401d2c26f65c89e3bbf83 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-afr_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_target: sentence_afr_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Afrikaans \nEnglish: {{sentence_eng_Latn}} \nAfrikaans: " +include: ntrex +task: ntrex_eng_Latn-afr_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-amh_Ethi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-amh_Ethi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5ae3dd1acdea962bb7f99533ddbe7255133a97b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-amh_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_target: sentence_amh_Ethi +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Amharic \nEnglish: {{sentence_eng_Latn}} \nAmharic: " +include: ntrex +task: ntrex_eng_Latn-amh_Ethi_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-arb_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-arb_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..303ccf471d5c180220c3985909377ba0227bbe41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-arb_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arb_Arab +doc_to_target: sentence_arb_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Arabic \nEnglish: {{sentence_eng_Latn}} \nArabic: " +include: ntrex +task: ntrex_eng_Latn-arb_Arab_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-bem_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-bem_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7992529aebb8b6ae9e2e55d2cfb89e142d047791 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-bem_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_target: sentence_bem_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Bemba \nEnglish: {{sentence_eng_Latn}} \nBemba: " +include: ntrex +task: ntrex_eng_Latn-bem_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ewe_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ewe_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3de8e8ebf4d0d15d24a0313d1793f26f7719167d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ewe_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_target: sentence_ewe_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Ewe \nEnglish: {{sentence_eng_Latn}} \nEwe: " +include: ntrex +task: ntrex_eng_Latn-ewe_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-fra_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-fra_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc230efe275e4712c7453f77d66290f44702b75d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-fra_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_target: sentence_fra_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to French \nEnglish: {{sentence_eng_Latn}} \nFrench: " +include: ntrex +task: ntrex_eng_Latn-fra_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-hau_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-hau_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..009d1a51893395026f0ed3d3f93e1a16c50abacc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-hau_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_target: sentence_hau_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Hausa \nEnglish: {{sentence_eng_Latn}} \nHausa: " +include: ntrex +task: ntrex_eng_Latn-hau_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ibo_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ibo_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2b27ab59197ab7a8b7069e83ac2186cac6d1510 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ibo_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_target: sentence_ibo_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Igbo \nEnglish: {{sentence_eng_Latn}} \nIgbo: " +include: ntrex +task: ntrex_eng_Latn-ibo_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-kin_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-kin_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f76077d3af55e07910b0c3aff74d400b3500b530 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-kin_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_target: sentence_kin_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kinyarwanda \nEnglish: {{sentence_eng_Latn}} \nKinyarwanda: " +include: ntrex +task: ntrex_eng_Latn-kin_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-mey_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-mey_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c5b2abaedbf8808a1ee5d15af9c3be837cbe63e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-mey_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mey_Arab +doc_to_target: sentence_mey_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Hassaniya Arabic \nEnglish: {{sentence_eng_Latn}} \nHassaniya Arabic: " +include: ntrex +task: ntrex_eng_Latn-mey_Arab_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-mlg_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-mlg_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d25afa1a04a13d8a3bf6e911ac151e7ac1da51f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-mlg_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mlg_Latn +doc_to_target: sentence_mlg_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Malagasy \nEnglish: {{sentence_eng_Latn}} \nMalagasy: " +include: ntrex +task: ntrex_eng_Latn-mlg_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-msa_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-msa_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7b7972bdaa207e0a34812496a40b8524da0305b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-msa_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: msa_Latn +doc_to_target: sentence_msa_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Malay \nEnglish: {{sentence_eng_Latn}} \nMalay: " +include: ntrex +task: ntrex_eng_Latn-msa_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nde_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nde_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31252c02f9f29312c18039903aa67f26e95499b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nde_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nde_Latn +doc_to_target: sentence_nde_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to North Ndebele \nEnglish: {{sentence_eng_Latn}} \nNorth Ndebele: " +include: ntrex +task: ntrex_eng_Latn-nde_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nso_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nso_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8daba4d8fc456e8a54fe14296d5762be002c3fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nso_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_target: sentence_nso_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Northern Sotho \nEnglish: {{sentence_eng_Latn}} \nNorthern Sotho: " +include: ntrex +task: ntrex_eng_Latn-nso_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nya_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nya_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe01ef879c5c52414bc39372103da5c5bff038fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nya_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_target: sentence_nya_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Chichewa \nEnglish: {{sentence_eng_Latn}} \nChichewa: " +include: ntrex +task: ntrex_eng_Latn-nya_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-orm_Ethi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-orm_Ethi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f78e4db60165e893242dea30d04503e5ae46ffb9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-orm_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm_Ethi +doc_to_target: sentence_orm_Ethi +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Oromo \nEnglish: {{sentence_eng_Latn}} \nOromo: " +include: ntrex +task: ntrex_eng_Latn-orm_Ethi_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-shi_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-shi_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57ea6c0480bc9bb9458cc4d6fa92215d67a518b0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-shi_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: shi_Arab +doc_to_target: sentence_shi_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tachelhit \nEnglish: {{sentence_eng_Latn}} \nTachelhit: " +include: ntrex +task: ntrex_eng_Latn-shi_Arab_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-sna_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-sna_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..399668d33c648f472ea6d980f8ebf2e659726b65 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-sna_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_target: sentence_sna_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Shona (Latin) \nEnglish: {{sentence_eng_Latn}} \nShona (Latin): " +include: ntrex +task: ntrex_eng_Latn-sna_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-som_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-som_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a29749aaeeda6c166b3a1cfecf843ef2f2ddfb3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-som_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_target: sentence_som_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Somali \nEnglish: {{sentence_eng_Latn}} \nSomali: " +include: ntrex +task: ntrex_eng_Latn-som_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ssw_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ssw_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a67ffdeee1465a9cd19b126e3a53a0e6ac054d05 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ssw_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_target: sentence_ssw_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Swati \nEnglish: {{sentence_eng_Latn}} \nSwati: " +include: ntrex +task: ntrex_eng_Latn-ssw_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-swa_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-swa_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0be54151da2d262039dd2c77753f0def8810e528 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-swa_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa_Latn +doc_to_target: sentence_swa_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Swahili \nEnglish: {{sentence_eng_Latn}} \nSwahili: " +include: ntrex +task: ntrex_eng_Latn-swa_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tam_Taml.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tam_Taml.yaml new file mode 100644 index 0000000000000000000000000000000000000000..387ac60dafe76aaf13adde1adb9830613172054a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tam_Taml.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tam_Taml +doc_to_target: sentence_tam_Taml +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tamil \nEnglish: {{sentence_eng_Latn}} \nTamil: " +include: ntrex +task: ntrex_eng_Latn-tam_Taml_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tel_Telu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tel_Telu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b1de396146f0c21caab39fea111fadfd53fce53 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tel_Telu.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tel_Telu +doc_to_target: sentence_tel_Telu +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Telugu \nEnglish: {{sentence_eng_Latn}} \nTelugu: " +include: ntrex +task: ntrex_eng_Latn-tel_Telu_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tir_Ethi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tir_Ethi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da402211718e0126e2281d32f1991c946b2a23fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tir_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_target: sentence_tir_Ethi +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tigrinya \nEnglish: {{sentence_eng_Latn}} \nTigrinya: " +include: ntrex +task: ntrex_eng_Latn-tir_Ethi_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ton_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ton_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8c466929bee40fb7ba2f5b000310925908251fd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ton_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ton_Latn +doc_to_target: sentence_ton_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tongan \nEnglish: {{sentence_eng_Latn}} \nTongan: " +include: ntrex +task: ntrex_eng_Latn-ton_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tsn_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tsn_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca918e1de6790bff10cabd103e546651643686d3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tsn_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_target: sentence_tsn_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tswana \nEnglish: {{sentence_eng_Latn}} \nTswana: " +include: ntrex +task: ntrex_eng_Latn-tsn_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-urd_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-urd_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8539df766542c9a7263ad62992b5fe619de2f23e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-urd_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: urd_Arab +doc_to_target: sentence_urd_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Urdu \nEnglish: {{sentence_eng_Latn}} \nUrdu: " +include: ntrex +task: ntrex_eng_Latn-urd_Arab_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ven_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ven_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e923b12ce695253b39965bda6352121271514123 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ven_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ven_Latn +doc_to_target: sentence_ven_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Venda \nEnglish: {{sentence_eng_Latn}} \nVenda: " +include: ntrex +task: ntrex_eng_Latn-ven_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-wol_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-wol_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..707b76a59f0bd3a661dfac59eb9413c46d323c8b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-wol_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_target: sentence_wol_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Wolof \nEnglish: {{sentence_eng_Latn}} \nWolof: " +include: ntrex +task: ntrex_eng_Latn-wol_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-xho_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-xho_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7f51491f88c75f9d2da270209fbe32b56bc529b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-xho_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_target: sentence_xho_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Xhosa \nEnglish: {{sentence_eng_Latn}} \nXhosa: " +include: ntrex +task: ntrex_eng_Latn-xho_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-yor_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-yor_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f3e4be543796276d04c65001199521701f02ed9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-yor_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_target: sentence_yor_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Yoruba \nEnglish: {{sentence_eng_Latn}} \nYoruba: " +include: ntrex +task: ntrex_eng_Latn-yor_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-zul_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-zul_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..946d0020ddb845653bc574e7cb8de54bf3a35a00 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-zul_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_target: sentence_zul_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Zulu \nEnglish: {{sentence_eng_Latn}} \nZulu: " +include: ntrex +task: ntrex_eng_Latn-zul_Latn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex new file mode 100644 index 0000000000000000000000000000000000000000..3bab54d824d83e7d201107a00411c22b5ec44a1b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex @@ -0,0 +1,25 @@ +tag: +- ntrex_afr-eng +- ntrex_afr-eng_prompt_3 +- afrobench_MT_tasks +dataset_path: masakhane/ntrex_african +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: test +fewshot_split: test +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_afr_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_afr_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09cbbfc56e84748c37d86366a68162b82869d918 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_afr_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Afrikaans and English linguist, translate the following Afrikaans\ + \ sentences to English \nAfrikaans: {{sentence_afr_Latn}}\nEnglish: " +include: ntrex +task: ntrex_afr_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_amh_Ethi-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_amh_Ethi-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33530440f5e232fa8b86267e8c42fab503d0c551 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_amh_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Amharic and English linguist, translate the following Amharic sentences\ + \ to English \nAmharic: {{sentence_amh_Ethi}}\nEnglish: " +include: ntrex +task: ntrex_amh_Ethi-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_arb_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_arb_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..858c0605f39ec13ecbed5733fad5d5eef3d275ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_arb_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arb_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Arabic and English linguist, translate the following Arabic sentences\ + \ to English \nArabic: {{sentence_arb_Arab}}\nEnglish: " +include: ntrex +task: ntrex_arb_Arab-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_bem_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_bem_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3be00855b6bfb5318442ebf3603c6c611f1c319c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_bem_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Bemba and English linguist, translate the following Bemba sentences\ + \ to English \nBemba: {{sentence_bem_Latn}}\nEnglish: " +include: ntrex +task: ntrex_bem_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ewe_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ewe_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..493176a7baed9c8b7ad64e6e028ce3b00d8a1067 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ewe_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Ewe and English linguist, translate the following Ewe sentences\ + \ to English \nEwe: {{sentence_ewe_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ewe_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_fra_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_fra_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b009a37bfd786707c077c55b391bacba7e6dad15 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_fra_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a French and English linguist, translate the following French sentences\ + \ to English \nFrench: {{sentence_fra_Latn}}\nEnglish: " +include: ntrex +task: ntrex_fra_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_hau_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_hau_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3c6f72111e504082a34674e05670288b6877d3c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_hau_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Hausa and English linguist, translate the following Hausa sentences\ + \ to English \nHausa: {{sentence_hau_Latn}}\nEnglish: " +include: ntrex +task: ntrex_hau_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ibo_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ibo_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4b7e768d4c4f4719842aa11af37f6afd89d4f9d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ibo_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Igbo and English linguist, translate the following Igbo sentences\ + \ to English \nIgbo: {{sentence_ibo_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ibo_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_kin_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_kin_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bba2d32395d22f86492720856e1801d586cab8ab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_kin_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kinyarwanda and English linguist, translate the following Kinyarwanda\ + \ sentences to English \nKinyarwanda: {{sentence_kin_Latn}}\nEnglish: " +include: ntrex +task: ntrex_kin_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_mey_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_mey_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a567548787c0b007306ef41762b9934eb1ad36e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_mey_Arab-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: mey_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Hassaniya Arabic and English linguist, translate the following\ + \ Hassaniya Arabic sentences to English \nHassaniya Arabic: {{sentence_mey_Arab}}\n\ + English: " +include: ntrex +task: ntrex_mey_Arab-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_mlg_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_mlg_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23d55c5a0b0f86bb04087ca00bb23c1970ad1fbd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_mlg_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mlg_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Malagasy and English linguist, translate the following Malagasy\ + \ sentences to English \nMalagasy: {{sentence_mlg_Latn}}\nEnglish: " +include: ntrex +task: ntrex_mlg_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_msa_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_msa_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa1a9618f29da1bbcc7171fda71629d593cada91 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_msa_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: msa_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Malay and English linguist, translate the following Malay sentences\ + \ to English \nMalay: {{sentence_msa_Latn}}\nEnglish: " +include: ntrex +task: ntrex_msa_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nde_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nde_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..855defd07be478e59de82c3eccb19e42dd07f042 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nde_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nde_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a North Ndebele and English linguist, translate the following North\ + \ Ndebele sentences to English \nNorth Ndebele: {{sentence_nde_Latn}}\nEnglish: " +include: ntrex +task: ntrex_nde_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nso_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nso_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29a7452c39392d1aa94bf3db22ed0ee9b62dd120 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nso_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Northern Sotho and English linguist, translate the following Northern\ + \ Sotho sentences to English \nNorthern Sotho: {{sentence_nso_Latn}}\nEnglish: " +include: ntrex +task: ntrex_nso_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nya_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nya_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..505586519ea0ea567364ae76d5db58fdec05da08 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nya_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Chichewa and English linguist, translate the following Chichewa\ + \ sentences to English \nChichewa: {{sentence_nya_Latn}}\nEnglish: " +include: ntrex +task: ntrex_nya_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_orm_Ethi-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_orm_Ethi-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a38e9312cdb30b6bc62b2d3f23c1e5583f043b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_orm_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Oromo and English linguist, translate the following Oromo sentences\ + \ to English \nOromo: {{sentence_orm_Ethi}}\nEnglish: " +include: ntrex +task: ntrex_orm_Ethi-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_shi_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_shi_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19f363ef2439a4986282116f9a34026205ebd431 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_shi_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: shi_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tachelhit and English linguist, translate the following Tachelhit\ + \ sentences to English \nTachelhit: {{sentence_shi_Arab}}\nEnglish: " +include: ntrex +task: ntrex_shi_Arab-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_sna_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_sna_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c7a63157eca8bdc6f8e2488fc1ef10b9941dbb9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_sna_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Shona (Latin) and English linguist, translate the following Shona\ + \ (Latin) sentences to English \nShona (Latin): {{sentence_sna_Latn}}\nEnglish: " +include: ntrex +task: ntrex_sna_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_som_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_som_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..685f38233c655048cb55819812247eefaea19527 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_som_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Somali and English linguist, translate the following Somali sentences\ + \ to English \nSomali: {{sentence_som_Latn}}\nEnglish: " +include: ntrex +task: ntrex_som_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ssw_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ssw_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd95665f6f17301764a1f1ad0d525352fd8f69bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ssw_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Swati and English linguist, translate the following Swati sentences\ + \ to English \nSwati: {{sentence_ssw_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ssw_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_swa_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_swa_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0731d37346153f0d9dec96c8211f7a8250ec3f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_swa_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Swahili and English linguist, translate the following Swahili sentences\ + \ to English \nSwahili: {{sentence_swa_Latn}}\nEnglish: " +include: ntrex +task: ntrex_swa_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tam_Taml-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tam_Taml-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..834320d846a40fcf6bd53c9445f051c38c38a439 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tam_Taml-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tam_Taml +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tamil and English linguist, translate the following Tamil sentences\ + \ to English \nTamil: {{sentence_tam_Taml}}\nEnglish: " +include: ntrex +task: ntrex_tam_Taml-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tel_Telu-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tel_Telu-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7fde743dcfd343546dcaa042bb3fef8a49a194d5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tel_Telu-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tel_Telu +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Telugu and English linguist, translate the following Telugu sentences\ + \ to English \nTelugu: {{sentence_tel_Telu}}\nEnglish: " +include: ntrex +task: ntrex_tel_Telu-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tir_Ethi-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tir_Ethi-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60189ee73207fc08911821188f61e23eb12dc62e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tir_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tigrinya and English linguist, translate the following Tigrinya\ + \ sentences to English \nTigrinya: {{sentence_tir_Ethi}}\nEnglish: " +include: ntrex +task: ntrex_tir_Ethi-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ton_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ton_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec2b5ba992a535f5f5f4fd6b269653b213f1b39a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ton_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ton_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tongan and English linguist, translate the following Tongan sentences\ + \ to English \nTongan: {{sentence_ton_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ton_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tsn_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tsn_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa63ca4b77edb7b0907e6660ce31df7ce0ea7278 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tsn_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tswana and English linguist, translate the following Tswana sentences\ + \ to English \nTswana: {{sentence_tsn_Latn}}\nEnglish: " +include: ntrex +task: ntrex_tsn_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_urd_Arab-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_urd_Arab-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b520795f2fd8e986b8292b985e36769c76f3553 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_urd_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: urd_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Urdu and English linguist, translate the following Urdu sentences\ + \ to English \nUrdu: {{sentence_urd_Arab}}\nEnglish: " +include: ntrex +task: ntrex_urd_Arab-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ven_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ven_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82372de2dd0624f9b068f27ab24f48433267ea28 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ven_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ven_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Venda and English linguist, translate the following Venda sentences\ + \ to English \nVenda: {{sentence_ven_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ven_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_wol_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_wol_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae0124f20efadbb363c017ef708b5dfb14311b07 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_wol_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Wolof and English linguist, translate the following Wolof sentences\ + \ to English \nWolof: {{sentence_wol_Latn}}\nEnglish: " +include: ntrex +task: ntrex_wol_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_xho_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_xho_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f0528af4efc5cb15035158a7c5789878eaa653b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_xho_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Xhosa and English linguist, translate the following Xhosa sentences\ + \ to English \nXhosa: {{sentence_xho_Latn}}\nEnglish: " +include: ntrex +task: ntrex_xho_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_yor_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_yor_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99d7cf494376be71148044b251c23c7b6f15191d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_yor_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Yoruba and English linguist, translate the following Yoruba sentences\ + \ to English \nYoruba: {{sentence_yor_Latn}}\nEnglish: " +include: ntrex +task: ntrex_yor_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_zul_Latn-eng_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_zul_Latn-eng_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30f3b307eef0a64f137cd993ff8571b103b2e91e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_zul_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Zulu and English linguist, translate the following Zulu sentences\ + \ to English \nZulu: {{sentence_zul_Latn}}\nEnglish: " +include: ntrex +task: ntrex_zul_Latn-eng_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex new file mode 100644 index 0000000000000000000000000000000000000000..d001e1f6e6acc14616603aa46a9f412d7abc026b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex @@ -0,0 +1,25 @@ +tag: +- ntrex_eng-afr +- ntrex_eng-afr_prompt_3 +- afrobench_MT_tasks +dataset_path: masakhane/ntrex_african +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: test +fewshot_split: test +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-afr_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-afr_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4aaa928ba0d31ca83a7d7eb59462a14715a2abf7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-afr_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_target: sentence_afr_Latn +doc_to_text: "As a Afrikaans and English linguist, translate the following English\ + \ sentences to Afrikaans \nEnglish: {{sentence_eng_Latn}} \nAfrikaans: " +include: ntrex +task: ntrex_eng_Latn-afr_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-amh_Ethi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-amh_Ethi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..008f73024a1c7136ba9c7db28badce24097da5d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-amh_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_target: sentence_amh_Ethi +doc_to_text: "As a Amharic and English linguist, translate the following English sentences\ + \ to Amharic \nEnglish: {{sentence_eng_Latn}} \nAmharic: " +include: ntrex +task: ntrex_eng_Latn-amh_Ethi_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-arb_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-arb_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0c9e8132374542c605789269c27aabf181dad28 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-arb_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arb_Arab +doc_to_target: sentence_arb_Arab +doc_to_text: "As a Arabic and English linguist, translate the following English sentences\ + \ to Arabic \nEnglish: {{sentence_eng_Latn}} \nArabic: " +include: ntrex +task: ntrex_eng_Latn-arb_Arab_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-bem_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-bem_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4ab2af30cba88d413b0c99a868b52614921aed8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-bem_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_target: sentence_bem_Latn +doc_to_text: "As a Bemba and English linguist, translate the following English sentences\ + \ to Bemba \nEnglish: {{sentence_eng_Latn}} \nBemba: " +include: ntrex +task: ntrex_eng_Latn-bem_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ewe_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ewe_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1c99ad06add81beb54d8b0e3b0d97a987bd2d70 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ewe_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_target: sentence_ewe_Latn +doc_to_text: "As a Ewe and English linguist, translate the following English sentences\ + \ to Ewe \nEnglish: {{sentence_eng_Latn}} \nEwe: " +include: ntrex +task: ntrex_eng_Latn-ewe_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-fra_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-fra_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3668db57aa9a1b9431dec109be78bf98e0080962 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-fra_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_target: sentence_fra_Latn +doc_to_text: "As a French and English linguist, translate the following English sentences\ + \ to French \nEnglish: {{sentence_eng_Latn}} \nFrench: " +include: ntrex +task: ntrex_eng_Latn-fra_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-hau_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-hau_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6bca042cb417c3511cf4e8fb442c61239f010a12 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-hau_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_target: sentence_hau_Latn +doc_to_text: "As a Hausa and English linguist, translate the following English sentences\ + \ to Hausa \nEnglish: {{sentence_eng_Latn}} \nHausa: " +include: ntrex +task: ntrex_eng_Latn-hau_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ibo_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ibo_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c23fcce8fee0b0767977b865af9f24eb27396384 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ibo_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_target: sentence_ibo_Latn +doc_to_text: "As a Igbo and English linguist, translate the following English sentences\ + \ to Igbo \nEnglish: {{sentence_eng_Latn}} \nIgbo: " +include: ntrex +task: ntrex_eng_Latn-ibo_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-kin_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-kin_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0041bfb4a44480f142af0c1b9ea37ccd9a47663 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-kin_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_target: sentence_kin_Latn +doc_to_text: "As a Kinyarwanda and English linguist, translate the following English\ + \ sentences to Kinyarwanda \nEnglish: {{sentence_eng_Latn}} \nKinyarwanda: " +include: ntrex +task: ntrex_eng_Latn-kin_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-mey_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-mey_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..435df83d9fe3e56a2a75cab98df058c54fd5a8a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-mey_Arab.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: mey_Arab +doc_to_target: sentence_mey_Arab +doc_to_text: "As a Hassaniya Arabic and English linguist, translate the following\ + \ English sentences to Hassaniya Arabic \nEnglish: {{sentence_eng_Latn}} \nHassaniya\ + \ Arabic: " +include: ntrex +task: ntrex_eng_Latn-mey_Arab_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-mlg_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-mlg_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74f92d92668d1b5a9539c503fd3aa5c687988ed8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-mlg_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mlg_Latn +doc_to_target: sentence_mlg_Latn +doc_to_text: "As a Malagasy and English linguist, translate the following English\ + \ sentences to Malagasy \nEnglish: {{sentence_eng_Latn}} \nMalagasy: " +include: ntrex +task: ntrex_eng_Latn-mlg_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-msa_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-msa_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc9a3245f365cdb7c03e5d67e45a9bb236b6477f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-msa_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: msa_Latn +doc_to_target: sentence_msa_Latn +doc_to_text: "As a Malay and English linguist, translate the following English sentences\ + \ to Malay \nEnglish: {{sentence_eng_Latn}} \nMalay: " +include: ntrex +task: ntrex_eng_Latn-msa_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nde_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nde_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7cf092447f290829c9ac3bbbcbb49d915543f26 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nde_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nde_Latn +doc_to_target: sentence_nde_Latn +doc_to_text: "As a North Ndebele and English linguist, translate the following English\ + \ sentences to North Ndebele \nEnglish: {{sentence_eng_Latn}} \nNorth Ndebele: " +include: ntrex +task: ntrex_eng_Latn-nde_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nso_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nso_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d52c1ef1f9c4a1ba82ae0f0722669fbf126569f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nso_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_target: sentence_nso_Latn +doc_to_text: "As a Northern Sotho and English linguist, translate the following English\ + \ sentences to Northern Sotho \nEnglish: {{sentence_eng_Latn}} \nNorthern Sotho: " +include: ntrex +task: ntrex_eng_Latn-nso_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nya_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nya_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a3d395516d48af64ed67d177cd0fa8b28fd9a46 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nya_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_target: sentence_nya_Latn +doc_to_text: "As a Chichewa and English linguist, translate the following English\ + \ sentences to Chichewa \nEnglish: {{sentence_eng_Latn}} \nChichewa: " +include: ntrex +task: ntrex_eng_Latn-nya_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-orm_Ethi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-orm_Ethi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3de07b02307696d09c97eec6120b69580dffade --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-orm_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm_Ethi +doc_to_target: sentence_orm_Ethi +doc_to_text: "As a Oromo and English linguist, translate the following English sentences\ + \ to Oromo \nEnglish: {{sentence_eng_Latn}} \nOromo: " +include: ntrex +task: ntrex_eng_Latn-orm_Ethi_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-shi_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-shi_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e193c7a3b5a73495c70dcd4176288a79cd6eb2c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-shi_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: shi_Arab +doc_to_target: sentence_shi_Arab +doc_to_text: "As a Tachelhit and English linguist, translate the following English\ + \ sentences to Tachelhit \nEnglish: {{sentence_eng_Latn}} \nTachelhit: " +include: ntrex +task: ntrex_eng_Latn-shi_Arab_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-sna_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-sna_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce8c50f5cacf69e70aca8f451ab8bc1fa8270158 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-sna_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_target: sentence_sna_Latn +doc_to_text: "As a Shona (Latin) and English linguist, translate the following English\ + \ sentences to Shona (Latin) \nEnglish: {{sentence_eng_Latn}} \nShona (Latin): " +include: ntrex +task: ntrex_eng_Latn-sna_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-som_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-som_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b7f46323401a4c04b1026507b1163111fa71455 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-som_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_target: sentence_som_Latn +doc_to_text: "As a Somali and English linguist, translate the following English sentences\ + \ to Somali \nEnglish: {{sentence_eng_Latn}} \nSomali: " +include: ntrex +task: ntrex_eng_Latn-som_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ssw_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ssw_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f02e88ca3f7d5abb314ea174fe21c35b48af402 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ssw_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_target: sentence_ssw_Latn +doc_to_text: "As a Swati and English linguist, translate the following English sentences\ + \ to Swati \nEnglish: {{sentence_eng_Latn}} \nSwati: " +include: ntrex +task: ntrex_eng_Latn-ssw_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-swa_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-swa_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47090821da435d1b9d4caada3e91221cd1eed3b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-swa_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa_Latn +doc_to_target: sentence_swa_Latn +doc_to_text: "As a Swahili and English linguist, translate the following English sentences\ + \ to Swahili \nEnglish: {{sentence_eng_Latn}} \nSwahili: " +include: ntrex +task: ntrex_eng_Latn-swa_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tam_Taml.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tam_Taml.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78d61866bd42b467246479946dfa342a6e7835ff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tam_Taml.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tam_Taml +doc_to_target: sentence_tam_Taml +doc_to_text: "As a Tamil and English linguist, translate the following English sentences\ + \ to Tamil \nEnglish: {{sentence_eng_Latn}} \nTamil: " +include: ntrex +task: ntrex_eng_Latn-tam_Taml_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tel_Telu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tel_Telu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..829635313dd6f5cbc1d08c31a52732aef1513e19 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tel_Telu.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tel_Telu +doc_to_target: sentence_tel_Telu +doc_to_text: "As a Telugu and English linguist, translate the following English sentences\ + \ to Telugu \nEnglish: {{sentence_eng_Latn}} \nTelugu: " +include: ntrex +task: ntrex_eng_Latn-tel_Telu_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tir_Ethi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tir_Ethi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f27f4389cb2b2be56b02f2427a8d8df222aed17 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tir_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_target: sentence_tir_Ethi +doc_to_text: "As a Tigrinya and English linguist, translate the following English\ + \ sentences to Tigrinya \nEnglish: {{sentence_eng_Latn}} \nTigrinya: " +include: ntrex +task: ntrex_eng_Latn-tir_Ethi_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ton_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ton_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ffeb74fbb04205e0bb1b27d0ec855252688f6e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ton_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ton_Latn +doc_to_target: sentence_ton_Latn +doc_to_text: "As a Tongan and English linguist, translate the following English sentences\ + \ to Tongan \nEnglish: {{sentence_eng_Latn}} \nTongan: " +include: ntrex +task: ntrex_eng_Latn-ton_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tsn_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tsn_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed11f2cba88a703b44ce2a077f761b4cd98135c0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tsn_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_target: sentence_tsn_Latn +doc_to_text: "As a Tswana and English linguist, translate the following English sentences\ + \ to Tswana \nEnglish: {{sentence_eng_Latn}} \nTswana: " +include: ntrex +task: ntrex_eng_Latn-tsn_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-urd_Arab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-urd_Arab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a05e951bef2f38005b9d7fb3133bdf811f69c565 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-urd_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: urd_Arab +doc_to_target: sentence_urd_Arab +doc_to_text: "As a Urdu and English linguist, translate the following English sentences\ + \ to Urdu \nEnglish: {{sentence_eng_Latn}} \nUrdu: " +include: ntrex +task: ntrex_eng_Latn-urd_Arab_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ven_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ven_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4345201694bc0c8a9f9bda487a9ecfb36982c8bf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ven_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ven_Latn +doc_to_target: sentence_ven_Latn +doc_to_text: "As a Venda and English linguist, translate the following English sentences\ + \ to Venda \nEnglish: {{sentence_eng_Latn}} \nVenda: " +include: ntrex +task: ntrex_eng_Latn-ven_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-wol_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-wol_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48abbb33f870ec3305f8337e62b131bbd38683fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-wol_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_target: sentence_wol_Latn +doc_to_text: "As a Wolof and English linguist, translate the following English sentences\ + \ to Wolof \nEnglish: {{sentence_eng_Latn}} \nWolof: " +include: ntrex +task: ntrex_eng_Latn-wol_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-xho_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-xho_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1071a5fb2faad74df4e2f357f412923162b0044 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-xho_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_target: sentence_xho_Latn +doc_to_text: "As a Xhosa and English linguist, translate the following English sentences\ + \ to Xhosa \nEnglish: {{sentence_eng_Latn}} \nXhosa: " +include: ntrex +task: ntrex_eng_Latn-xho_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-yor_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-yor_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43c1be35ee76adf853e6429e4bb06fea867ce5d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-yor_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_target: sentence_yor_Latn +doc_to_text: "As a Yoruba and English linguist, translate the following English sentences\ + \ to Yoruba \nEnglish: {{sentence_eng_Latn}} \nYoruba: " +include: ntrex +task: ntrex_eng_Latn-yor_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-zul_Latn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-zul_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10e890a9b3cbffdbb2205d091d91fa42eae880b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-zul_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_target: sentence_zul_Latn +doc_to_text: "As a Zulu and English linguist, translate the following English sentences\ + \ to Zulu \nEnglish: {{sentence_eng_Latn}} \nZulu: " +include: ntrex +task: ntrex_eng_Latn-zul_Latn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/README.md b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fe980e87464b07c91d2c766254c760d772d65c36 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/README.md @@ -0,0 +1,25 @@ +# + +## Paper +Title: `Multilingual Massive Multitask Language Understanding (MMMLU)` + +Paper Link: https://arxiv.org/abs/2009.03300 + +## Abstract +>We propose a new test to measure a text model's multitask accuracy. The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more. To attain high accuracy on this test, models must possess extensive world knowledge and problem solving ability. We find that while most recent models have near random-chance accuracy, the very largest GPT-3 model improves over random chance by almost 20 percentage points on average. However, on every one of the 57 tasks, the best models still need substantial improvements before they can reach expert-level accuracy. Models also have lopsided performance and frequently do not know when they are wrong. Worse, they still have near-random accuracy on some socially important subjects such as morality and law. By comprehensively evaluating the breadth and depth of a model's academic and professional understanding, our test can be used to analyze models across many tasks and to identify important shortcomings. + +HomePage: https://huggingface.co/datasets/openai/MMMLU + +### Citation + +``` +@misc{hendrycks2021measuringmassivemultitasklanguage, + title={Measuring Massive Multitask Language Understanding}, + author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt}, + year={2021}, + eprint={2009.03300}, + archivePrefix={arXiv}, + primaryClass={cs.CY}, + url={https://arxiv.org/abs/2009.03300}, +} +``` diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/openai_mmlu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/openai_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..541eb43cfdd783b15cad4123437c2dffcf1cc794 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/openai_mmlu.yaml @@ -0,0 +1,13 @@ +group: openai_mmlu +task: + - openai_mmlu_prompt_1 + - openai_mmlu_prompt_2 + - openai_mmlu_prompt_3 + - openai_mmlu_prompt_4 + - openai_mmlu_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu new file mode 100644 index 0000000000000000000000000000000000000000..ce4f02eeda277404713974f4699c716b454514f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu @@ -0,0 +1,22 @@ +tag: + - openai_mmlu_tasks + - openai_mmlu_prompt_1 + - afrobench_mmlu_tasks +dataset_path: openai/MMMLU +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer.strip())}}" +should_decontaminate: true +doc_to_decontamination_query: "{{Question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_ara.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_ara.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c9b86fc1d1c5d8185692c48bc85d991714dbff5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_ara.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: AR_XY +doc_to_text: 'Q: {{Question.strip()}} + + A: {{A}} + + B: {{B}} + + C: {{C}} + + D: {{D}} + + Please choose the correct answer from the options above:' +include: openai_mmlu +task: openai_mmlu_ara_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a3661d45235258f0c0cb1a6bb21119de326ef7f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_swa.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: SW_KE +doc_to_text: 'Q: {{Question.strip()}} + + A: {{A}} + + B: {{B}} + + C: {{C}} + + D: {{D}} + + Please choose the correct answer from the options above:' +include: openai_mmlu +task: openai_mmlu_swa_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4124252bfc0b549160ac802f18c44004792d3bf2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_yor.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: YO_NG +doc_to_text: 'Q: {{Question.strip()}} + + A: {{A}} + + B: {{B}} + + C: {{C}} + + D: {{D}} + + Please choose the correct answer from the options above:' +include: openai_mmlu +task: openai_mmlu_yor_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu new file mode 100644 index 0000000000000000000000000000000000000000..9f39b0a9d7423b4d5638f23f294b636240570281 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu @@ -0,0 +1,22 @@ +tag: + - openai_mmlu_tasks + - openai_mmlu_prompt_2 + - afrobench_mmlu_tasks +dataset_path: openai/MMMLU +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer.strip())}}" +should_decontaminate: true +doc_to_decontamination_query: "{{Question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_ara.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_ara.yaml new file mode 100644 index 0000000000000000000000000000000000000000..550834257a69f7054ae397a403c6dc00d15c8888 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_ara.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: AR_XY +doc_to_text: 'Question: {{Question.strip()}} + + 1: {{A}} + + 2: {{B}} + + 3: {{C}} + + 4: {{D}} + + Please select the correct answer from the given choices:' +include: openai_mmlu +task: openai_mmlu_ara_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b3025fd726ab59f48ee90bb65b294580a6cfc3c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_swa.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: SW_KE +doc_to_text: 'Question: {{Question.strip()}} + + 1: {{A}} + + 2: {{B}} + + 3: {{C}} + + 4: {{D}} + + Please select the correct answer from the given choices:' +include: openai_mmlu +task: openai_mmlu_swa_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..145b237ef50234278732605b1e3936bfccb9968a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_yor.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: YO_NG +doc_to_text: 'Question: {{Question.strip()}} + + 1: {{A}} + + 2: {{B}} + + 3: {{C}} + + 4: {{D}} + + Please select the correct answer from the given choices:' +include: openai_mmlu +task: openai_mmlu_yor_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu new file mode 100644 index 0000000000000000000000000000000000000000..95456656739a2490a3e11037e7d9f67f72d60962 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu @@ -0,0 +1,23 @@ +tag: + - openai_mmlu_tasks + - openai_mmlu_prompt_3 + - afrobench_mmlu_tasks +dataset_path: openai/MMMLU +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer.strip())}}" +should_decontaminate: true +doc_to_decontamination_query: "{{Question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_ara.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_ara.yaml new file mode 100644 index 0000000000000000000000000000000000000000..012192ceee6638f197db8ea8b9210e1529b6b92d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_ara.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: AR_XY +doc_to_text: 'Input Question: {{Question.strip()}} + + Option A: {{A}} + + Option B: {{B}} + + Option C: {{C}} + + Option D: {{D}} + + Please indicate the correct option from the list above:' +include: openai_mmlu +task: openai_mmlu_ara_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..431bdb345178bf44b12ea01507cc805cd000113f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_swa.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: SW_KE +doc_to_text: 'Input Question: {{Question.strip()}} + + Option A: {{A}} + + Option B: {{B}} + + Option C: {{C}} + + Option D: {{D}} + + Please indicate the correct option from the list above:' +include: openai_mmlu +task: openai_mmlu_swa_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..814fe380267e57da691f727198f2828042aa54c0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_yor.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: YO_NG +doc_to_text: 'Input Question: {{Question.strip()}} + + Option A: {{A}} + + Option B: {{B}} + + Option C: {{C}} + + Option D: {{D}} + + Please indicate the correct option from the list above:' +include: openai_mmlu +task: openai_mmlu_yor_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu new file mode 100644 index 0000000000000000000000000000000000000000..37a5949f93795737f8f61a06fc2824ebb671dbe2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu @@ -0,0 +1,23 @@ +tag: + - openai_mmlu_tasks + - openai_mmlu_prompt_4 + - afrobench_mmlu_tasks +dataset_path: openai/MMMLU +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer.strip())}}" +should_decontaminate: true +doc_to_decontamination_query: "{{Question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_ara.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_ara.yaml new file mode 100644 index 0000000000000000000000000000000000000000..793eb7441ce36573953525c3c97e60daffb10b02 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_ara.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: AR_XY +doc_to_text: 'Critically analyze the question and select the most probable answer + from the list: + + {{Question.strip()}} + + Choices: + + A) {{A}} + + B) {{B}} + + C) {{C}} + + D) {{D}}' +include: openai_mmlu +task: openai_mmlu_ara_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..095dd7ff6d04db581bc070eff001d48600014e0e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_swa.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: SW_KE +doc_to_text: 'Critically analyze the question and select the most probable answer + from the list: + + {{Question.strip()}} + + Choices: + + A) {{A}} + + B) {{B}} + + C) {{C}} + + D) {{D}}' +include: openai_mmlu +task: openai_mmlu_swa_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd0a9daa1ed5a2ae9882225aefdfe5e653dffcc6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_yor.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: YO_NG +doc_to_text: 'Critically analyze the question and select the most probable answer + from the list: + + {{Question.strip()}} + + Choices: + + A) {{A}} + + B) {{B}} + + C) {{C}} + + D) {{D}}' +include: openai_mmlu +task: openai_mmlu_yor_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu new file mode 100644 index 0000000000000000000000000000000000000000..77183eb04c0567b83f87bfd17bbdd18bf003f7dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu @@ -0,0 +1,23 @@ +tag: + - openai_mmlu_tasks + - openai_mmlu_prompt_5 + - afrobench_mmlu_tasks +dataset_path: openai/MMMLU +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer.strip())}}" +should_decontaminate: true +doc_to_decontamination_query: "{{Question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_ara.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_ara.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50a6e74ff2198b326716b99a7430102d8aaf0221 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_ara.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: AR_XY +doc_to_text: 'Answer the question and pick the correct answer from the options: {{Question.strip()}} + + Options: + + A. {{A}} + + B. {{B}} + + C. {{C}} + + D. {{D}} + + Please choose the correct option from the above list:' +include: openai_mmlu +task: openai_mmlu_ara_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0cc19860cc5f7bc90d499a1ead811a549170eb6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_swa.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: SW_KE +doc_to_text: 'Answer the question and pick the correct answer from the options: {{Question.strip()}} + + Options: + + A. {{A}} + + B. {{B}} + + C. {{C}} + + D. {{D}} + + Please choose the correct option from the above list:' +include: openai_mmlu +task: openai_mmlu_swa_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..691657ef46974107e46c37291fb1efa66364a5b9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_yor.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: YO_NG +doc_to_text: 'Answer the question and pick the correct answer from the options: {{Question.strip()}} + + Options: + + A. {{A}} + + B. {{B}} + + C. {{C}} + + D. {{D}} + + Please choose the correct option from the above list:' +include: openai_mmlu +task: openai_mmlu_yor_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0fc0fea958c32b2b8d104f586045564c04de8c86 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/openai_mmlu/utils.py @@ -0,0 +1,99 @@ +import argparse +import os + +import yaml + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_1": "Q: {{Question.strip()}}\nA: {{A}}\nB: {{B}}\nC: {{C}}\nD: {{D}}\nPlease choose the correct answer from the options above:", + "prompt_2": "Question: {{Question.strip()}}\n1: {{A}}\n2: {{B}}\n3: {{C}}\n4: {{D}}\nPlease select the correct answer from the given choices:", + "prompt_3": "Input Question: {{Question.strip()}}\nOption A: {{A}}\nOption B: {{B}}\nOption C: {{C}}\nOption D: {{D}}\nPlease indicate the correct option from the list above:", + "prompt_4": "Critically analyze the question and select the most probable answer from the list:\n{{Question.strip()}}\nChoices:\nA) {{A}}\nB) {{B}}\nC) {{C}}\nD) {{D}}", + "prompt_5": "Answer the question and pick the correct answer from the options: {{Question.strip()}}\nOptions:\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nPlease choose the correct option from the above list:", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "ara": "Arabic", + "swa": "Swahili", + "yor": "Yoruba", + } + + lang2_code = { + "ara": "AR_XY", + "swa": "SW_KE", + "yor": "YO_NG", + } + + for lang in languages.keys(): + try: + file_name = f"openai_mmlu_{lang}.yaml" + task_name = f"openai_mmlu_{lang}_{mode}" + yaml_template = "openai_mmlu" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang2_code[lang], + "doc_to_text": prompt_func(mode, languages[lang]), + } + file_path = os.path.join(output_dir, mode) + os.makedirs(file_path, exist_ok=True) + + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_1", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/README.md b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3c5239a05e88cbfbadf6670f96d6ed621b0d805c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/README.md @@ -0,0 +1,17 @@ +# + +## Paper +Title: `Sunbird African Language Technology (SALT) dataset` + +Paper Link: https://aclanthology.org/2023.emnlp-main.862/ + +## Abstract +>SALT is a multi-way parallel text and speech corpus of Engish and six languages widely spoken in Uganda and East Africa: Luganda, Lugbara, Acholi, Runyankole, Ateso and Swahili. The core of the dataset is a set of 25,000 sentences covering a range of topics of local relevance, such as agriculture, health and society. Each sentence is translated into all languages, to support machine translation, and speech recordings are made for approximately 5,000 of the sentences both by a variety of speakers in natural settings (suitable for ASR) and by professionals in a studio setting (suitable for text-to-speech). + +HomePage: https://github.com/SunbirdAI/salt + +### Publications + +Multilingual Model and Data Resources for Text-To-Speech in Ugandan Languages. Isaac Owomugisha, Benjamin Akera, Ernest Tonny Mwebaze, John Quinn. 4th Workshop on African Natural Language Processing, 2023. [pdf](https://openreview.net/pdf?id=vaxG0WAPzL) + +Machine Translation For African Languages: Community Creation Of Datasets And Models In Uganda. Benjamin Akera, Jonathan Mukiibi, Lydia Sanyu Naggayi, Claire Babirye, Isaac Owomugisha, Solomon Nsumba, Joyce Nakatumba-Nabende, Engineer Bainomugisha, Ernest Mwebaze, John Quinn. 3rd Workshop on African Natural Language Processing, 2022. [pdf](https://openreview.net/pdf?id=BK-z5qzEU-9) diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/gen_utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/gen_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6ac703a0d5d0912d38fb624dbba967ed3ffdb734 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/gen_utils.py @@ -0,0 +1,149 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang, lang_dict): + language_column_name = f"{lang}_text" + prompt_map = { + "prompt_1": f"{lang_dict[lang]} sentence: {{{{{language_column_name}}}}} \nEnglish sentence: ", + "prompt_1_reverse": "English sentence: {{eng_source_text}} " + f"\n{lang_dict[lang]} sentence: ", + "prompt_2": f"You are a translation expert. Translate the following {lang_dict[lang]} sentences to English \n" + f"{lang_dict[lang]} sentence: {{{{{language_column_name}}}}}\nEnglish sentence: ", + "prompt_2_reverse": f"You are a translation expert. Translate the following English sentences to " + f"{lang_dict[lang]} " + "\nEnglish sentence: {{eng_source_text}} " + f"\n{lang_dict[lang]} sentence: ", + "prompt_3": f"As a {lang_dict[lang]} and English linguist, translate the following {lang_dict[lang]} sentences " + f"to English. \n{lang_dict[lang]} sentence: {{{{{language_column_name}}}}}\nEnglish sentence: ", + "prompt_3_reverse": f"As a {lang_dict[lang]} and English linguist, translate the following English sentences to " + f"{lang_dict[lang]}. " + "\nEnglish sentence: {{eng_source_text}} " + f"\n{lang_dict[lang]} sentence: ", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str, reverse: bool) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "eng": "English", + "lug": "Luganda", + "ach": "Acholi", + "lgg": "Lugbara", + "teo": "Ateso", + "nyn": "Runyankole", + "swa": "Swahili", + "ibo": "Igbo", + } + + for lang in languages.keys(): + try: + if lang != "eng": + if not reverse: + file_name = f"salt_{lang}-eng.yaml" + task_name = f"salt_{lang}-eng_{mode}" + yaml_template = "salt" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": "text-all", + "doc_to_target": "eng_target_text", + "doc_to_text": prompt_func(mode, lang, languages), + } + os.makedirs(f"{output_dir}/{mode}", exist_ok=True) + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + else: + file_name = f"salt_eng-{lang}.yaml" + task_name = f"salt_eng-{lang}_{mode}" + yaml_template = "salt" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": "text-all", + "doc_to_target": f"{lang}_text", + "doc_to_text": prompt_func(f"{mode}_reverse", lang, languages), + } + os.makedirs(f"{output_dir}/{mode}", exist_ok=True) + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_1", + choices=["prompt_1", "prompt_2", "prompt_3"], + help="Prompt number", + ) + parser.add_argument( + "--reverse", + default=True, + choices=[True, False], + help="Reverse the translation direction", + ) + args = parser.parse_args() + + gen_lang_yamls( + output_dir=args.output_dir, + overwrite=args.overwrite, + mode=args.mode, + reverse=args.reverse, + ) + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt new file mode 100644 index 0000000000000000000000000000000000000000..a07d434a8bfb5e4c85abef6fe556e648c6fe5a00 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt @@ -0,0 +1,24 @@ +tag: +- salt_tasks +- salt_prompt_1 +- afrobench_MT_tasks +dataset_path: Sunbird/salt +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_ach-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_ach-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41731279817637401307fc9f55ecd96cd2a80794 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_ach-eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "Acholi sentence: {{ach_text}} \nEnglish sentence: " +include: salt +task: salt_ach-eng_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-ach.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-ach.yaml new file mode 100644 index 0000000000000000000000000000000000000000..219e5780634f4812157ea6d2ad70b7b22e72ae49 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-ach.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: ach_text +doc_to_text: "English sentence: {{eng_source_text}} \nAcholi sentence: " +include: salt +task: salt_eng-ach_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f90220591f5f7047da6d488740c759c850a95b1e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: ibo_text +doc_to_text: "English sentence: {{eng_source_text}} \nIgbo sentence: " +include: salt +task: salt_eng-ibo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-lgg.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-lgg.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a038ddb39eb1b171be9e5631e129995ceeed64e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-lgg.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: lgg_text +doc_to_text: "English sentence: {{eng_source_text}} \nLugbara sentence: " +include: salt +task: salt_eng-lgg_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4539913786124aec4ea68f16538989a91131ca44 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-lug.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: lug_text +doc_to_text: "English sentence: {{eng_source_text}} \nLuganda sentence: " +include: salt +task: salt_eng-lug_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-nyn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-nyn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..448e1101d681d4f31bde8c81418d4f2f64b6eb13 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-nyn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: nyn_text +doc_to_text: "English sentence: {{eng_source_text}} \nRunyankole sentence: " +include: salt +task: salt_eng-nyn_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..792b4840c2551627b66008fdd2c172e3660cc914 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-swa.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: swa_text +doc_to_text: "English sentence: {{eng_source_text}} \nSwahili sentence: " +include: salt +task: salt_eng-swa_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-teo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-teo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..810626c6a5ddf8525d45344b5a5eb7a2d65ab34e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-teo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: teo_text +doc_to_text: "English sentence: {{eng_source_text}} \nAteso sentence: " +include: salt +task: salt_eng-teo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_ibo-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_ibo-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a98c8648081bc8c3e1fd1c897c41212701f36fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_ibo-eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "Igbo sentence: {{ibo_text}} \nEnglish sentence: " +include: salt +task: salt_ibo-eng_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_lgg-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_lgg-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8e281ac3189dbaada8d21fbd4896a0c8478dbc6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_lgg-eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "Lugbara sentence: {{lgg_text}} \nEnglish sentence: " +include: salt +task: salt_lgg-eng_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_lug-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_lug-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f924d5c50f82e1dbbf6be1dd4a138d2c5d61c5ac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_lug-eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "Luganda sentence: {{lug_text}} \nEnglish sentence: " +include: salt +task: salt_lug-eng_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_nyn-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_nyn-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd9363614648969391f20deb49fd2a92afdcfede --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_nyn-eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "Runyankole sentence: {{nyn_text}} \nEnglish sentence: " +include: salt +task: salt_nyn-eng_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_swa-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_swa-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2308593e3d54e222d7543403f214dba76719a80 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_swa-eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "Swahili sentence: {{swa_text}} \nEnglish sentence: " +include: salt +task: salt_swa-eng_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_teo-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_teo-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6efb4ea0807a9a66eb84797503b6bc4762777fd0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_1/salt_teo-eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "Ateso sentence: {{teo_text}} \nEnglish sentence: " +include: salt +task: salt_teo-eng_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt new file mode 100644 index 0000000000000000000000000000000000000000..66355878cbb8354261bd426623d29589ce93383a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt @@ -0,0 +1,24 @@ +tag: +- salt_tasks +- salt_prompt_2 +- afrobench_MT_tasks +dataset_path: Sunbird/salt +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_ach-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_ach-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dda717b7942cb37c7f6d821070572cd302717639 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_ach-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "You are a translation expert. Translate the following Acholi sentences\ + \ to English \nAcholi sentence: {{ach_text}}\nEnglish sentence: " +include: salt +task: salt_ach-eng_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-ach.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-ach.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e4a72a5116a41a7d7950cfed80cbd826a37a0dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-ach.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: ach_text +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Acholi \nEnglish sentence: {{eng_source_text}} \nAcholi sentence: " +include: salt +task: salt_eng-ach_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04649c1287e599a2ecdf376b4b30bc86700dcaca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: ibo_text +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Igbo \nEnglish sentence: {{eng_source_text}} \nIgbo sentence: " +include: salt +task: salt_eng-ibo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-lgg.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-lgg.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ac6becbcb7b10890cb1b2cd56dbe43c23742683 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-lgg.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: lgg_text +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Lugbara \nEnglish sentence: {{eng_source_text}} \nLugbara sentence: " +include: salt +task: salt_eng-lgg_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b5f6399cf6ddc5276fb48545e4ad3d1e0e4ab1f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: lug_text +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Luganda \nEnglish sentence: {{eng_source_text}} \nLuganda sentence: " +include: salt +task: salt_eng-lug_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-nyn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-nyn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84452d5aed07b2fe13d6836a7656ff85dfa2ae8e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-nyn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: nyn_text +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Runyankole \nEnglish sentence: {{eng_source_text}} \nRunyankole sentence: " +include: salt +task: salt_eng-nyn_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..523db9fb7e913dff30b80d28ca13b8c613653ad6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: swa_text +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Swahili \nEnglish sentence: {{eng_source_text}} \nSwahili sentence: " +include: salt +task: salt_eng-swa_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-teo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-teo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..000e8d043bb1897c5647480d6584191181b45c68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-teo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: teo_text +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Ateso \nEnglish sentence: {{eng_source_text}} \nAteso sentence: " +include: salt +task: salt_eng-teo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_ibo-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_ibo-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4ec6601af313b25606c05752715a3dfadf1476e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_ibo-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "You are a translation expert. Translate the following Igbo sentences\ + \ to English \nIgbo sentence: {{ibo_text}}\nEnglish sentence: " +include: salt +task: salt_ibo-eng_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_lgg-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_lgg-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d802c0faa99f895605e00c25aea3197b0fad7d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_lgg-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "You are a translation expert. Translate the following Lugbara sentences\ + \ to English \nLugbara sentence: {{lgg_text}}\nEnglish sentence: " +include: salt +task: salt_lgg-eng_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_lug-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_lug-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..521bbf15c008670a0d71b671be84e58b9ca7290b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_lug-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "You are a translation expert. Translate the following Luganda sentences\ + \ to English \nLuganda sentence: {{lug_text}}\nEnglish sentence: " +include: salt +task: salt_lug-eng_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_nyn-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_nyn-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4cc4abfc26505ada2abb05775a6b4b43c67fb139 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_nyn-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "You are a translation expert. Translate the following Runyankole sentences\ + \ to English \nRunyankole sentence: {{nyn_text}}\nEnglish sentence: " +include: salt +task: salt_nyn-eng_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_swa-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_swa-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e80b9087df91df24f619c61c61c553decfcb1bf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_swa-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "You are a translation expert. Translate the following Swahili sentences\ + \ to English \nSwahili sentence: {{swa_text}}\nEnglish sentence: " +include: salt +task: salt_swa-eng_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_teo-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_teo-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0b0d516de9ae00758cd9fccb45c84d65eb069bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_2/salt_teo-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "You are a translation expert. Translate the following Ateso sentences\ + \ to English \nAteso sentence: {{teo_text}}\nEnglish sentence: " +include: salt +task: salt_teo-eng_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt new file mode 100644 index 0000000000000000000000000000000000000000..51dac9c53b42569b2b5c7f19a5b9fa6b83fc68e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt @@ -0,0 +1,24 @@ +tag: +- salt_tasks +- salt_prompt_3 +- afrobench_MT_tasks +dataset_path: Sunbird/salt +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_ach-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_ach-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c198a59f843447475f221823361f7ddf919419c3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_ach-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "As a Acholi and English linguist, translate the following Acholi sentences\ + \ to English. \nAcholi sentence: {{ach_text}}\nEnglish sentence: " +include: salt +task: salt_ach-eng_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-ach.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-ach.yaml new file mode 100644 index 0000000000000000000000000000000000000000..636a77d8606343d9de230547b958f8e49b448b5c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-ach.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: ach_text +doc_to_text: "As a Acholi and English linguist, translate the following English sentences\ + \ to Acholi. \nEnglish sentence: {{eng_source_text}} \nAcholi sentence: " +include: salt +task: salt_eng-ach_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..44d015d6ca9db85477a082c687a46c7e46276068 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: ibo_text +doc_to_text: "As a Igbo and English linguist, translate the following English sentences\ + \ to Igbo. \nEnglish sentence: {{eng_source_text}} \nIgbo sentence: " +include: salt +task: salt_eng-ibo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-lgg.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-lgg.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f1e6f43ba7783c2b521ee3a0caeec1d0904790e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-lgg.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: lgg_text +doc_to_text: "As a Lugbara and English linguist, translate the following English sentences\ + \ to Lugbara. \nEnglish sentence: {{eng_source_text}} \nLugbara sentence: " +include: salt +task: salt_eng-lgg_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2065c30df12a680ca08b218ce3e842324313da4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: lug_text +doc_to_text: "As a Luganda and English linguist, translate the following English sentences\ + \ to Luganda. \nEnglish sentence: {{eng_source_text}} \nLuganda sentence: " +include: salt +task: salt_eng-lug_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-nyn.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-nyn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e48970a8ccb136d4328598224c370076949954b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-nyn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: nyn_text +doc_to_text: "As a Runyankole and English linguist, translate the following English\ + \ sentences to Runyankole. \nEnglish sentence: {{eng_source_text}} \nRunyankole\ + \ sentence: " +include: salt +task: salt_eng-nyn_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfd3f8eadb1ca0ff898595c897a3eebde72f08a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: swa_text +doc_to_text: "As a Swahili and English linguist, translate the following English sentences\ + \ to Swahili. \nEnglish sentence: {{eng_source_text}} \nSwahili sentence: " +include: salt +task: salt_eng-swa_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-teo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-teo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8d280bb41808f8af50287861fa1131b92295e70 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-teo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: teo_text +doc_to_text: "As a Ateso and English linguist, translate the following English sentences\ + \ to Ateso. \nEnglish sentence: {{eng_source_text}} \nAteso sentence: " +include: salt +task: salt_eng-teo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_ibo-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_ibo-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13be699cb1dc1255939321205d25921625cdb140 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_ibo-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "As a Igbo and English linguist, translate the following Igbo sentences\ + \ to English. \nIgbo sentence: {{ibo_text}}\nEnglish sentence: " +include: salt +task: salt_ibo-eng_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_lgg-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_lgg-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7aa4ffc442c41ea9abd148257b1e49524173eca5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_lgg-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "As a Lugbara and English linguist, translate the following Lugbara sentences\ + \ to English. \nLugbara sentence: {{lgg_text}}\nEnglish sentence: " +include: salt +task: salt_lgg-eng_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_lug-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_lug-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da505f6d7589d9a7bba4ea7be1c73134fc562a20 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_lug-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "As a Luganda and English linguist, translate the following Luganda sentences\ + \ to English. \nLuganda sentence: {{lug_text}}\nEnglish sentence: " +include: salt +task: salt_lug-eng_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_nyn-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_nyn-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9edba7c495369e1849106e854100a65d0bda9ee5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_nyn-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "As a Runyankole and English linguist, translate the following Runyankole\ + \ sentences to English. \nRunyankole sentence: {{nyn_text}}\nEnglish sentence: " +include: salt +task: salt_nyn-eng_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_swa-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_swa-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d01c9170c602c7eebdc3b0a5c216d5bdd4bc52a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_swa-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "As a Swahili and English linguist, translate the following Swahili sentences\ + \ to English. \nSwahili sentence: {{swa_text}}\nEnglish sentence: " +include: salt +task: salt_swa-eng_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_teo-eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_teo-eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c81336cac58f12d6dd2118315a6cdb64a913a2af --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/prompt_3/salt_teo-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "As a Ateso and English linguist, translate the following Ateso sentences\ + \ to English. \nAteso sentence: {{teo_text}}\nEnglish sentence: " +include: salt +task: salt_teo-eng_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/salt.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/salt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..edd3070d8ba2c24b651038ca7408a38b45e00da3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/salt/salt.yaml @@ -0,0 +1,11 @@ +group: salt +task: + - salt_prompt_1 + - salt_prompt_2 + - salt_prompt_3 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sample_run_scripts/run_afrobench.sh b/lm-evaluation-harness/lm_eval/tasks/afrobench/sample_run_scripts/run_afrobench.sh new file mode 100644 index 0000000000000000000000000000000000000000..886c94956cc8204ce9fda69e912cec91424a3d92 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sample_run_scripts/run_afrobench.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +batch_size=5 +num_fewshot=0 + +export CUDA_VISIBLE_DEVICES=0,1 + +model_names=( + "google/gemma-1.1-7b-it", + "google/gemma-2-9b-it", + "google/gemma-2-27b-it", + "Jacaranda/AfroLlama_V1", + "LLaMAX/LLaMAX3-8B-Alpaca", + "meta-llama/Llama-2-7b-chat-hf", + "meta-llama/Llama-3.1-8B-Instruct", + "meta-llama/Llama-3.1-70B-Instruct", + "meta-llama/Meta-Llama-3-8B-Instruct", + "CohereForAI/aya-101" +) + +for model_name in "${model_names[@]}" +do + echo "Running model: $model_name" + lm_eval --model hf \ + --model_args pretrained=${model_names},parallelize=true \ + --tasks afrobench\ + --batch_size ${batch_size} \ + --num_fewshot ${num_fewshot} \ + --verbosity DEBUG \ + --output_path 'path_to_results/' \ + --log_samples +done diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sample_run_scripts/run_afrobench_lite.sh b/lm-evaluation-harness/lm_eval/tasks/afrobench/sample_run_scripts/run_afrobench_lite.sh new file mode 100644 index 0000000000000000000000000000000000000000..89291faadb97fa9267d09be80e81a7b480aabcb5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sample_run_scripts/run_afrobench_lite.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +batch_size=5 +num_fewshot=0 + +export CUDA_VISIBLE_DEVICES=0,1 + +model_names=( + "google/gemma-1.1-7b-it", + "google/gemma-2-9b-it", + "google/gemma-2-27b-it", + "Jacaranda/AfroLlama_V1", + "LLaMAX/LLaMAX3-8B-Alpaca", + "meta-llama/Llama-2-7b-chat-hf", + "meta-llama/Llama-3.1-8B-Instruct", + "meta-llama/Llama-3.1-70B-Instruct", + "meta-llama/Meta-Llama-3-8B-Instruct", + "CohereForAI/aya-101" +) + +for model_name in "${model_names[@]}" +do + echo "Running model: $model_name" + lm_eval --model hf \ + --model_args pretrained=${model_name},parallelize=true \ + --tasks afrobench_lite\ + --batch_size ${batch_size} \ + --num_fewshot ${num_fewshot} \ + --verbosity DEBUG \ + --output_path 'path_to_results/' \ + --log_samples +done diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/README.md b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/README.md new file mode 100644 index 0000000000000000000000000000000000000000..732db84b0eb6ad373442692b221e7f97e18e112a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/README.md @@ -0,0 +1,37 @@ +# + +## Paper +Title: `SIB-200: A Simple, Inclusive, and Big Evaluation Dataset for Topic Classification in 200+ Languages and Dialects` + +Paper Link: https://aclanthology.org/2024.eacl-long.14/ + +## Abstract +>Despite the progress in building multilingual language models, evaluation is often limited to a few languages with available datasets which excludes a large number of low-resource languages. In this paper, we create SIB-200—a large-scale open-sourced benchmark dataset for topic classification in 205 languages and dialects to address the lack of evaluation dataset for Natural Language Understanding (NLU). For many of the languages covered in SIB-200, this is the first publicly available evaluation dataset for NLU. The dataset is based on Flores-200 machine translation corpus. We annotated the English portion of the dataset and extended the sentence-level annotation to the remaining 204 languages covered in the corpus. Despite the simplicity of this task, our evaluation in full-supervised setting, cross-lingual transfer setting and prompting of large language model setting show that there is still a large gap between the performance of high-resource and low-resource languages when multilingual evaluation is scaled to numerous world languages. We found that languages unseen during the pre-training of multilingual language models, languages from under-represented families (like Nilotic and Altantic-Congo), and languages from the regions of Africa, Americas, Oceania and South East Asia, often have the lowest performance on our topic classification dataset. We hope our dataset %will encourages a more inclusive evaluation of multilingual language models on a more diverse set of languages. + +HomePage: https://github.com/dadelani/sib-200 + +### Citation + +``` +@inproceedings{adelani-etal-2024-sib, + title = "{SIB}-200: A Simple, Inclusive, and Big Evaluation Dataset for Topic Classification in 200+ Languages and Dialects", + author = "Adelani, David Ifeoluwa and + Liu, Hannah and + Shen, Xiaoyu and + Vassilyev, Nikita and + Alabi, Jesujoba O. and + Mao, Yanke and + Gao, Haonan and + Lee, En-Shiun Annie", + editor = "Graham, Yvette and + Purver, Matthew", + booktitle = "Proceedings of the 18th Conference of the European Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = mar, + year = "2024", + address = "St. Julian{'}s, Malta", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2024.eacl-long.14/", + pages = "226--245", + abstract = "Despite the progress in building multilingual language models, evaluation is often limited to a few languages with available datasets which excludes a large number of low-resource languages. In this paper, we create SIB-200{---}a large-scale open-sourced benchmark dataset for topic classification in 205 languages and dialects to address the lack of evaluation dataset for Natural Language Understanding (NLU). For many of the languages covered in SIB-200, this is the first publicly available evaluation dataset for NLU. The dataset is based on Flores-200 machine translation corpus. We annotated the English portion of the dataset and extended the sentence-level annotation to the remaining 204 languages covered in the corpus. Despite the simplicity of this task, our evaluation in full-supervised setting, cross-lingual transfer setting and prompting of large language model setting show that there is still a large gap between the performance of high-resource and low-resource languages when multilingual evaluation is scaled to numerous world languages. We found that languages unseen during the pre-training of multilingual language models, languages from under-represented families (like Nilotic and Altantic-Congo), and languages from the regions of Africa, Americas, Oceania and South East Asia, often have the lowest performance on our topic classification dataset. We hope our dataset {\%}will encourages a more inclusive evaluation of multilingual language models on a more diverse set of languages." +} +``` diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib new file mode 100644 index 0000000000000000000000000000000000000000..37fda5d192dc8b4e1aa115d66858876e6bca3bda --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib @@ -0,0 +1,43 @@ +tag: + - sib_tasks + - sib_prompt_1 + - afrobench_TC_tasks +dataset_path: Davlan/sib200 +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: category +doc_to_choice: + - "science/technology" + - "travel" + - "politics" + - "sports" + - "health" + - "entertainment" + - "geography" +should_decontaminate: true +doc_to_decontamination_query: text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_aeb.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_aeb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4116035df2599f79d31293b25abf43191943abd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_aeb.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aeb_Arab +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_aeb_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_afr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_afr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..001eee846bd92a3e1703d64d799e5bc8c066f70e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_afr.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_afr_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_aka.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_aka.yaml new file mode 100644 index 0000000000000000000000000000000000000000..907977dc638bdfc7aba5ea11324d54667cd21d1e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_aka.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aka_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_aka_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dde5420724bdb678ac877c5ff895df74ba0b08c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_amh_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_ary.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_ary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68347bd51439c95b88403f843fb78a06a3562d39 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_ary.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_ary_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_arz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_arz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c0328134c766bd56637a2097f1b87bfa03a4973 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_arz.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_arz_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5469a8a17ea44b468172c326a148f1185a559015 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_bam.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_bam_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_bem.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_bem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01aaa1cbd82342de4ace8c11387f1851a21661d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_bem.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_bem_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_cjk.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_cjk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6deaee753f460189a1fcf47c800239b2242ccf8d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_cjk.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: cjk_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_cjk_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_dik.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_dik.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d80d0a080890269475d0133cb4a73cc80ffbe6eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_dik.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: dik_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_dik_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_dyu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_dyu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d72e6321e92d7e8947cce5109d363f7eb51f9de --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_dyu.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: dyu_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_dyu_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e32469681e4400517131926dff6e8b1a717b69d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_eng_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60cf7db830a1aff9215989f4af5c9a6f8d278985 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_ewe.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_ewe_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_fon.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_fon.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ae765522ccd81ddadd2842bb7e8a346fff18088 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_fon.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fon_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_fon_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4614e6d27f2d41e5558045d933df41a66a909cfb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_fra.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_fra_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_fuv.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_fuv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24f1d28a8f088d383bf7fbbff939dc73b4cf447e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_fuv.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_fuv_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_gaz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_gaz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df904f957318c08eaf8c2f5cba4d0befa5220fbc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_gaz.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_gaz_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b160b8cfc0aa662bfadcc68f2891208e7039c01b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_hau_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e481aeacd5c7d63cbfd11e7efcb3fb1ac738e945 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_ibo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a914b01cc54c35941cd769dbe6667ee624421b91 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kab_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kab_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aaa05108b0cc3313932e71a174b0f53e747e42eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kam.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kam_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kam_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kbp.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kbp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d428490863c573b3a757672bc3c074d8fd548c0d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kbp.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kbp_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kbp_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kea.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kea.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e458fb225b8c1b4b4ee2823f46b4b4ad7a6dcad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kea.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kea_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kik.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kik.yaml new file mode 100644 index 0000000000000000000000000000000000000000..beb94a8edb7cec7b51c960fe319a98e798a84581 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kik.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kik_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kik_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c16432eba27800bcc8eb927e4a201aac7b3f2e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kin_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kmb.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kmb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c46477e31e4639a9b9c1dca0ce59318534e883e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kmb.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kmb_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kmb_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_knc.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_knc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b43157e3642dc91b6f04de06ecc622b67fb036e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_knc.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: knc_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_knc_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kon.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kon.yaml new file mode 100644 index 0000000000000000000000000000000000000000..def4a77def17ff2d11cc00d6c87962a03c4081cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_kon.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kon_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kon_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbba95e0cf7217c4385f4601a7867ffc6576b2b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_lin_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_lua.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_lua.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4bc665b3f9e6fce1703b2ea53c93bcc52111363 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_lua.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lua_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_lua_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbf42e1889e695a379d8261bac27d02fa7f4d33d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_lug_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a62ea03c7ba534928d5c3c333d631216cf0dd248 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_luo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_luo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_mos.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_mos.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54140a5d1339758f59a3504d3a4a0a5448414b90 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_mos.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mos_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_mos_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_nso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_nso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f7382d58f3071f1dddcab360ecf06b2cf7a427c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_nso.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_nso_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_nus.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_nus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28208912f85036d26b494ed495b43f2a57982869 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_nus.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nus_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_nus_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_nya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_nya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ca90a9233e68301406a3303ebbb85cb47da2207 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_nya.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_nya_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_plt.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_plt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..650b9a4b711f30b59c0aadde797e941d69a17ed6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_plt.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_plt_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_por.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_por.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7901e924a043d74dadbf8b0dabff2303273d03a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_por.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_por_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_run.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_run.yaml new file mode 100644 index 0000000000000000000000000000000000000000..510fc5c15841c9c130af5cce0e3e2d8499eb71d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_run.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: run_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_run_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_sag.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_sag.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7c0bb3148857edab4e8eaef00974fa5e4dfd974 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_sag.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sag_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_sag_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4115112c393c0dd424b14bdd66046d58e82eb83 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_sna_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_som.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_som.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be9c19f1039b8093e3c5bcd7573168b23f6e923c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_som.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_som_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78d0e1f50dc0909475131e7892bbe726f5144412 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_sot.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_sot_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_ssw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_ssw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..988f6828cbe84bdf7cec2a03798a452956e1768d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_ssw.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_ssw_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4a92192eb750ed34c647381ae0c8655b141f4a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_swa_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_taq.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_taq.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a860f019dd0d259ea3fd9eddfb776870ea24b7f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_taq.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: taq_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_taq_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..606755c5c59c01cdd1148437cdfccb4791ebc689 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_tir.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_tir_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_tso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_tso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6b2e46369554e76573e3cdec7128b56d9853913 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_tso.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_tso_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_tum.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_tum.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e17521fb63ca03a4b38747157cf0171dcb2cf13 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_tum.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tum_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_tum_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf818808af1a80bfa7cfad46b5f16930b5619636 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_twi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: twi_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_twi_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_tzm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_tzm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10cf4c5b6626fe9ffc3addfbc8197156e23ee45f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_tzm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tzm_Tfng +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_tzm_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_umb.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_umb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d171c9c6b6fd7f2db5ac205e3adfed2e6e6fb867 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_umb.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: umb_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_umb_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3a6d7e6234c0ac9df872fb3cfcbc1e9f0e4f483 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_wol.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_wol_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57ce4d2db833add543832e1798a90b7479d8a360 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_xho.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_xho_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cab811762f3b61828cb698857e0d46f33855f568 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_yor_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..694ddfc11f55e33b04544bde8a2004939e8bb158 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/sib_zul.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_zul_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_1/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib new file mode 100644 index 0000000000000000000000000000000000000000..27dd7d1f64838b9692fbaa06ea98c6cd7f7db97e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib @@ -0,0 +1,43 @@ +tag: + - sib_tasks + - sib_prompt_2 + - afrobench_TC_tasks +dataset_path: Davlan/sib200 +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: category +doc_to_choice: + - "science/technology" + - "travel" + - "politics" + - "sports" + - "health" + - "entertainment" + - "geography" +should_decontaminate: true +doc_to_decontamination_query: text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_aeb.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_aeb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32b2443948fd04761dab4331d9421b50e6293397 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_aeb.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: aeb_Arab +doc_to_text: 'Does this Tunisian Arabic topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_aeb_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_afr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_afr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c212b13f1f2cf4cd9a2b5b70fce75429a2dbbd91 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_afr.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: 'Does this Afrikaans topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_afr_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_aka.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_aka.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dacfef07608dfebc67025bc8ff983260ee535f6b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_aka.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: aka_Latn +doc_to_text: 'Does this Akan topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_aka_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..259009f056b82ff8968feb9df082f7c232845124 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_amh.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: 'Does this Amharic topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_amh_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_ary.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_ary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..141a6691de71e7f932970dd3a73c91aa818c45b2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_ary.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: 'Does this Moroccan Arabic topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_ary_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_arz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_arz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2fee5eed9e22ca4448a5aa1efe26e756ed41562 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_arz.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: 'Does this Egyptian Arabic topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_arz_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ae5ddd0ea44b3d4c9a90e40125b942cd1919d26 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_bam.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: 'Does this Bambara topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_bam_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_bem.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_bem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1631a349226b60b9de250b4f97db8e474094951e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_bem.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_text: 'Does this Bemba topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_bem_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_cjk.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_cjk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85521f131a3532d7791bc3c022572bb624fd653c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_cjk.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: cjk_Latn +doc_to_text: 'Does this Chokwe topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_cjk_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_dik.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_dik.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c660516f42e0e869c8a266d113e65dcbbbf8f032 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_dik.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: dik_Latn +doc_to_text: 'Does this Southwestern Dinka topic; ''{{text}}'' belong to one of the + following categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_dik_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_dyu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_dyu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..698782fda2a65ea766eef9b91381d497949005ca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_dyu.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: dyu_Latn +doc_to_text: 'Does this Dyula topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_dyu_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..564d86565f8aa47d9944d3a5aedc9555ca29c9a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_eng.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: 'Does this English topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_eng_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba064082941553b1177d6c4ea4901e6aa7ba61be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_ewe.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_text: 'Does this Ewe topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_ewe_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_fon.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_fon.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9bb542dd84452cdd500d01a6e561c408e7a7fcf1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_fon.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: fon_Latn +doc_to_text: 'Does this Fon topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_fon_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf279d611378a2a1981415940a692389728fd339 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_fra.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: 'Does this French topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_fra_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_fuv.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_fuv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50bb4b824748d070ad7d004efd15d2bab5cd8c0f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_fuv.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: 'Does this Nigerian Fulfulde topic; ''{{text}}'' belong to one of the + following categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_fuv_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_gaz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_gaz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..601d5f79f2605a3c0db8278500ecce1f5987222a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_gaz.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: 'Does this West Central Oromo topic; ''{{text}}'' belong to one of the + following categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_gaz_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c7255d4747d4b5a033129bca1be441114bef36f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_hau.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: 'Does this Hausa topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_hau_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..714c132f655a0b57e9c160a513a6c73350c5919c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_ibo.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: 'Does this Igbo topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_ibo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22303a3fbb1db518170ee57c258cff95c9f2c134 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kab.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kab_Latn +doc_to_text: 'Does this Kabyle topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_kab_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..498781d6e836f9854a7703f669e80b3a16003637 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kam.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kam_Latn +doc_to_text: 'Does this Kamba topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_kam_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kbp.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kbp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..679d7ccd7a74430df674154fa03af065fc4e23a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kbp.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kbp_Latn +doc_to_text: 'Does this Kabiye topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_kbp_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kea.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kea.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aee33cf27faf2fed8b6873b601a5a437dae11bb7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kea.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: 'Does this Kabuverdianu topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_kea_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kik.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kik.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77c87bc131b912e0564156acf740cb5aa3007615 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kik.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kik_Latn +doc_to_text: 'Does this Kikuyu topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_kik_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5be0643e11f39513363994cc6bbc02ac1604f24c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kin.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: 'Does this Kinyarwanda topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_kin_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kmb.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kmb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02f4e9d22410d932c345df2eeb1b4de1c3e71c4b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kmb.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kmb_Latn +doc_to_text: 'Does this Kimbundu topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_kmb_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_knc.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_knc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2623c480235bbf269b082a6604af129bb82e7df4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_knc.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: knc_Latn +doc_to_text: 'Does this Central Kanuri topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_knc_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kon.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kon.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ec3bcf97652bde14ee764bf961ea49aca088df4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_kon.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kon_Latn +doc_to_text: 'Does this Kikongo topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_kon_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec2fa57a8bbc4309bbb44a865568a2cf70b842e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_lin.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: 'Does this Lingala topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_lin_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_lua.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_lua.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f3acc3dbeb2b708257c9b5f1fcc7cac4a703d54 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_lua.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: lua_Latn +doc_to_text: 'Does this Luba-Kasai topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_lua_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d6e7b9f0c315b7868580704abc3cdba0775cc65 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_lug.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: 'Does this Luganda topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_lug_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d1a438594b1818e3fe34c9ce63e47e0c802e700 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_luo.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: 'Does this Luo topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_luo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_mos.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_mos.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc025905e76696e820804e4989e9b1bec2fa2257 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_mos.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: mos_Latn +doc_to_text: 'Does this Mossi topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_mos_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_nso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_nso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75021cc514b64598cd1e94902ecdd653db50681e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_nso.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_text: 'Does this Northern Sotho topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_nso_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_nus.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_nus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..abca40e85705feeaae8fb118ae9d162c611e0545 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_nus.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: nus_Latn +doc_to_text: 'Does this Nuer topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_nus_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_nya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_nya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e09e27331ad6104a7c585f63638b4e24e9ba8880 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_nya.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: 'Does this Nyanga topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_nya_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_plt.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_plt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5b385cade643f329904a7a0dab0797e57433581 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_plt.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: 'Does this Plateau Malagasy topic; ''{{text}}'' belong to one of the + following categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_plt_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_por.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_por.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a297c05a4be4992f33fb07737bd704ab076c9cfe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_por.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: 'Does this Portuguese topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_por_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_run.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_run.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4bb32245653846c6eb82fef3716b31ba85adf4d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_run.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: run_Latn +doc_to_text: 'Does this Rundi topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_run_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_sag.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_sag.yaml new file mode 100644 index 0000000000000000000000000000000000000000..979b4d84e0dae472a83fdb64e7b62c35453763e9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_sag.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: sag_Latn +doc_to_text: 'Does this Sango topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_sag_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b41184b3c702c2346484fb6a982a1f9a10fe6516 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_sna.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: 'Does this Shona topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_sna_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_som.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_som.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cda1fb4133df8f42bf69a0e296b866cd128ef368 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_som.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: 'Does this Somali topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_som_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08d0dbecbec823c712108c42d64f9a7cbed73463 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_sot.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: 'Does this Southern Sotho topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_sot_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_ssw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_ssw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d3b99e7a07affa07aeb7ad4452887808fbd47de --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_ssw.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: 'Does this Swazi topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_ssw_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e9faa831698a196843ae2f6b5f8cf4939bdada0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_swa.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: 'Does this Swahili topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_swa_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_taq.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_taq.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1862c468c6c874e56fca81c9bbc0df09c91425e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_taq.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: taq_Latn +doc_to_text: 'Does this Tamasheq topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_taq_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80dcc1bb3d8d6a65ea6dcdf75af3c946a803b071 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_tir.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: 'Does this Tigrinya topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_tir_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_tso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_tso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fad909b4b7a714ba44df14041808d54e2dc7edc2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_tso.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: 'Does this Tsonga topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_tso_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_tum.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_tum.yaml new file mode 100644 index 0000000000000000000000000000000000000000..613535bc95647f5edf818e844423eebad5291937 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_tum.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: tum_Latn +doc_to_text: 'Does this Tumbuka topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_tum_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..064edb4cb8e0e1a88dbc1ccfad20adefa13034e9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_twi.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: twi_Latn +doc_to_text: 'Does this Twi topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_twi_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_tzm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_tzm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ec8adc260622a611261d30aebd49450d202b700 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_tzm.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: tzm_Tfng +doc_to_text: 'Does this Tamazight topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_tzm_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_umb.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_umb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a910abc5fa30c9462c950f059059f5f91554b3a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_umb.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: umb_Latn +doc_to_text: 'Does this Umbundu topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_umb_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4453b3458ecde8bbc26cff73793db71471301850 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_wol.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: 'Does this Wolof topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_wol_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e038cc9458fbb376331296bd4dd1c96a5f26a8f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_xho.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: 'Does this Xhosa topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_xho_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e831b3117b828bb4ea68016ca19fdcd9c89525b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_yor.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: 'Does this Yoruba topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_yor_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f61a4061f2b636167400491e59db890289aff3d3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/sib_zul.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: 'Does this Zulu topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_zul_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_2/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib new file mode 100644 index 0000000000000000000000000000000000000000..fed4e5c5019f791c72cfbe214efb2698943c5b92 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib @@ -0,0 +1,43 @@ +tag: + - sib_tasks + - sib_prompt_3 + - afrobench_TC_tasks +dataset_path: Davlan/sib200 +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: category +doc_to_choice: + - "science/technology" + - "travel" + - "politics" + - "sports" + - "health" + - "entertainment" + - "geography" +should_decontaminate: true +doc_to_decontamination_query: text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_aeb.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_aeb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b82cc4ec3cc8cff2dff2818f9238477ea12528a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_aeb.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: aeb_Arab +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Tunisian Arabic statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_aeb_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_afr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_afr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f818759646be15a6c6d1c0193a7b24deb730bb03 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_afr.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Afrikaans statement below? Return only\ + \ the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_afr_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_aka.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_aka.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d4ff4e42cf1d7c926015812ea5c716928b697fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_aka.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: aka_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Akan statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_aka_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58207e9e39010c4f30d3ff0a1f46fd9e53f3b042 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_amh.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Amharic statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_amh_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_ary.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_ary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ccb9a06880d6aa946ffd750429da3fb650c46eea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_ary.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Moroccan Arabic statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_ary_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_arz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_arz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19ebbed7b9a441cca520eff58a663354d29a7395 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_arz.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Egyptian Arabic statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_arz_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d2c1a18d9b3a4b076ca70b32981b2ecb58e3f9c0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_bam.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Bambara statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_bam_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_bem.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_bem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99750497c258c93b1393ced6f34b9a724fbe518d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_bem.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Bemba statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_bem_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_cjk.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_cjk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..470612b51b1f7d4821573764692a04f2a623a42f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_cjk.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: cjk_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Chokwe statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_cjk_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_dik.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_dik.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5269b0262805239b807d789a7343dc0d1507a29 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_dik.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: dik_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Southwestern Dinka statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_dik_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_dyu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_dyu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f04a1c17199e50ecebfac887459b3c3f124a1529 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_dyu.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: dyu_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Dyula statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_dyu_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf22d08fcab877aae2ce77081274d1928e27f8c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_eng.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the English statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_eng_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4cc991048ad19285b1dd269e91a6bb32898b6d88 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_ewe.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Ewe statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_ewe_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_fon.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_fon.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3127fde242956bcaf89224ed3a88be80dc967c52 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_fon.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: fon_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Fon statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_fon_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a24ff30e4f6408f02a0f4a8978250d91e36621f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_fra.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the French statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_fra_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_fuv.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_fuv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..405838c78ddbe6a99d66f436699de00f0b7e814b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_fuv.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Nigerian Fulfulde statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_fuv_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_gaz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_gaz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..282b439a3c2d6703c04446b4cf477a9bd60bf340 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_gaz.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the West Central Oromo statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_gaz_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..239181bf1f1586ac83aa1741e9cffe531c2433b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_hau.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Hausa statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_hau_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0581291dd7ed5cf53e52fe2f44154363b8be9599 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_ibo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Igbo statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_ibo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32fbbf4407d07a5d78558b022260117f592645d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kab.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kab_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kabyle statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kab_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f745ba54f9daaca1a3302443c4a5aba3de795f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kam.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kam_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kamba statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kam_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kbp.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kbp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5be1035bb58862fe73c2678287620d409a76b87 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kbp.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kbp_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kabiye statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kbp_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kea.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kea.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1d3e2a68cf8b465f589701f1004ae4b5dc07dd9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kea.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kabuverdianu statement below? Return only\ + \ the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kea_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kik.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kik.yaml new file mode 100644 index 0000000000000000000000000000000000000000..521a0f89226460e6f1a9e25c0d24066cd929c662 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kik.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kik_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kikuyu statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kik_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..066bfb630c59e334f65dcd74ea536a6790b3337d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kin.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kinyarwanda statement below? Return only\ + \ the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kin_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kmb.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kmb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c548af893d77f13231cab13318e396cbcf423388 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kmb.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kmb_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kimbundu statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kmb_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_knc.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_knc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9136823770a1e6754070143b2b0e40a988da22f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_knc.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: knc_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Central Kanuri statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_knc_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kon.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kon.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8777511ef33fad4019d3e157d1dbc4f6d0aad96 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_kon.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kon_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kikongo statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kon_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8688cb875fa5554625073325810a9dbb1198f06b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_lin.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Lingala statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_lin_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_lua.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_lua.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e71ac2aae77f40f06796c1572b2d38b44ec53962 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_lua.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lua_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Luba-Kasai statement below? Return only\ + \ the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_lua_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3554267ebad03a604a4a3dcca369af535efb156 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_lug.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Luganda statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_lug_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..161814d36bde3e61fe3cdf38e98d2ef62f6b9248 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_luo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Luo statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_luo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_mos.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_mos.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b80d5008087bf66a82b8b7855fc5b8c857497fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_mos.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: mos_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Mossi statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_mos_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_nso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_nso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c9dd8bd3f8cd30ce172286ca005c16b0ead9214 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_nso.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Northern Sotho statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_nso_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_nus.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_nus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..361698af10f6f231fbbdabf9e92a287504c45057 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_nus.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: nus_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Nuer statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_nus_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_nya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_nya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c455c788ad7fc13a440885b6c7fc594ed4fc6e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_nya.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Nyanga statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_nya_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_plt.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_plt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb90a034be0e94aba92823ba3b5762fc13eabe6f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_plt.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Plateau Malagasy statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_plt_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_por.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_por.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65b8c2da4ab91e1723ceabd2e9fb08d3b6de2cfe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_por.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Portuguese statement below? Return only\ + \ the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_por_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_run.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_run.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19f3681cf856c7bc28bb1fcb5e8c31eda1f1b618 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_run.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: run_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Rundi statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_run_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_sag.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_sag.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dfdcbd41929bc5747c34f18e93633fec8ac04e6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_sag.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: sag_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Sango statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_sag_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f30ff0d2b995c831b4005318a18a723998c92aa8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_sna.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Shona statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_sna_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_som.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_som.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ea27fd2e1b298324e1e1abcff152b25cd9cfc3d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_som.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Somali statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_som_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4ad477db4c912a09e317c7edc7818fe96b355f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_sot.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Southern Sotho statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_sot_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_ssw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_ssw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25b7f85e1c955207a6afe5d154ed4286602a5313 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_ssw.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Swazi statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_ssw_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7be0be9d211d0fce16493b3b62d593a7ad60b864 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_swa.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Swahili statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_swa_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_taq.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_taq.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7e7b3abbbbca622c1b56169a7abc6c917d9b241 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_taq.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: taq_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Tamasheq statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_taq_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aceb352596ba9fed4c4a3a544beb616923dca213 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_tir.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Tigrinya statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_tir_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_tso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_tso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..025b7163c069a6291dc4691a34de44732cc4c8b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_tso.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Tsonga statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_tso_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_tum.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_tum.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35092ea79435767ad3e4907e152273d2cd6f1dca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_tum.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: tum_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Tumbuka statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_tum_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc75f6579cdb57391a21bbce3a49fb062d7263f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_twi.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: twi_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Twi statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_twi_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_tzm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_tzm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9b3044cdd08f81c95edfaeb8ded07d4a1da919d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_tzm.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: tzm_Tfng +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Tamazight statement below? Return only\ + \ the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_tzm_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_umb.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_umb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8bb8540180f44e612ea82e5276749d104362492 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_umb.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: umb_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Umbundu statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_umb_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..115796d5276b8739efb6fccfb9064b5f4bb6a27e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_wol.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Wolof statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_wol_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b61c84b700da4e798848d52ed2310c9cb5ee3467 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_xho.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Xhosa statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_xho_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5ccd0c738eb5d1d365e5d7342ebe4eaaf7686b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_yor.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Yoruba statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_yor_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4915989dbbb0849b3345a80a420daa18a37eb97b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/sib_zul.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Zulu statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_zul_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_3/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib new file mode 100644 index 0000000000000000000000000000000000000000..28ed8f4a0da4e25815ebcfa6e58092a382e1708e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib @@ -0,0 +1,43 @@ +tag: + - sib_tasks + - sib_prompt_4 + - afrobench_TC_tasks +dataset_path: Davlan/sib200 +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: category +doc_to_choice: + - "science/technology" + - "travel" + - "politics" + - "sports" + - "health" + - "entertainment" + - "geography" +should_decontaminate: true +doc_to_decontamination_query: text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_aeb.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_aeb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e8c737f278122c8893e028ea2334ff93646a73cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_aeb.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aeb_Arab +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_aeb_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_afr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_afr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7139d04e9a5b4a4865ba11941d3078802cc9a85c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_afr.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_afr_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_aka.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_aka.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59c8c56a6b78ffbb0da5e3b3abeb24ccd13b35d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_aka.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aka_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_aka_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cec6b6c43425195e36be88cbdc266d8806844a24 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_amh_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_ary.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_ary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c10743470b814bf689bfef10410af3b4e03bb84 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_ary.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_ary_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_arz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_arz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1740975a66196d9c4c3bd6780ad50281766cb0b2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_arz.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_arz_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33ee240e6d95a0e43426b514f5e33f696526faeb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_bam.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_bam_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_bem.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_bem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa5608e849606f6d63f371b1bd7362d355b7d42b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_bem.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_bem_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_cjk.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_cjk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52e08d7b8c5dc10d6c35a5b4fa4deee9b494f2d5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_cjk.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: cjk_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_cjk_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_dik.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_dik.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8db6013f1d2a63d8242ac59d55c3f006f01e660 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_dik.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: dik_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_dik_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_dyu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_dyu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9bbc0b547f3b25150eb000d4e56bb6e24e86991 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_dyu.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: dyu_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_dyu_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c84749120e002dee47446d05600d81ed14bc193 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_eng_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02e7ea822fee11a3d0b3869ef3ea493048a114da --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_ewe.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_ewe_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_fon.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_fon.yaml new file mode 100644 index 0000000000000000000000000000000000000000..67053ed8cd739682270062acea206792a7df5679 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_fon.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fon_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_fon_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2b858ce4e502334f8440c8551b1bcd10feb3b15 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_fra.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_fra_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_fuv.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_fuv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c73f82679a48664a468f8e36432a0e33399190c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_fuv.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_fuv_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_gaz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_gaz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba95ef5d8ee884a65befdab1a83853686d8b8ef5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_gaz.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_gaz_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d53794868c164768810226db74aab7f06ccb383 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_hau_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2683d98dba0ca644b5314ef96a1359571a83fe9d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_ibo_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f645a4598e2de1ccc45de14274a882b26deceb7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kab_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kab_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f035b89505f2f6ef889addc2af1c972efc8ff2d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kam.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kam_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kam_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kbp.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kbp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c65b6352e1dd12d2a7d511825f9c043ad213aebc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kbp.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kbp_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kbp_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kea.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kea.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e7bba4ae7a6c359568f6252edaebd0bee96c860 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kea.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kea_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kik.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kik.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06480d183bb1d2765f8d23e8dda80ee6c37c029e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kik.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kik_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kik_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b447219fb3bafaf2a81e3ac727e5216f408893f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kin_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kmb.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kmb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5fc51890964f59d20f53b06cf3ddbdb02b444471 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kmb.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kmb_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kmb_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_knc.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_knc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..326443318488c921765094a96c84adb8e208eda8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_knc.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: knc_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_knc_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kon.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kon.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6da4ab390d0e433391e313dea2c82d302d090dd2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_kon.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kon_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kon_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51076dbd56131d82389d83bf2b12a224ef6c5443 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_lin_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_lua.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_lua.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95973f7d5ce8309ee595ce4e751d564851796923 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_lua.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lua_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_lua_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a570b58488496b4a73ca0fe46a2210c95b470bb2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_lug_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76d799856b98c4f78804815fd5bd86dd415a100d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_luo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_luo_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_mos.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_mos.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aeb058ac584c862898e7acf7441aa76b8c123709 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_mos.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mos_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_mos_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_nso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_nso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f76e016a6bcdda14daf24179da982f696732a199 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_nso.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_nso_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_nus.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_nus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..255c1861589e185c2bcdb3f1d9f679ed26837be0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_nus.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nus_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_nus_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_nya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_nya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc7a48abf7f86cdea8f07d54a3a166ffe9550f06 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_nya.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_nya_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_plt.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_plt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..880c3d03ef5be9db726024243158f283c2013861 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_plt.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_plt_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_por.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_por.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16a258365d56b100c44fd269da27413fd3bffa83 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_por.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_por_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_run.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_run.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a97737edf4b9ab31a53749d109359c8acb3d3f4f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_run.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: run_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_run_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_sag.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_sag.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c6897795ec414f37037eb2b79e6ffb6e3124ed7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_sag.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sag_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_sag_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da13a6ecf2b11650068be21b5b28ff470aba9002 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_sna_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_som.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_som.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6c35f3cb4a25a3ddecda9d4b1dc8528fce64d1f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_som.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_som_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1385e058deaf19b0cdf272a768f260997d7cae92 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_sot.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_sot_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_ssw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_ssw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d678c12422ff05f87849f934daf402454fa3415e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_ssw.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_ssw_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7492cfa329f48c959f6255ffbb879d952fcbe200 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_swa_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_taq.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_taq.yaml new file mode 100644 index 0000000000000000000000000000000000000000..783be833f8c77c16a82948f4055162941723849f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_taq.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: taq_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_taq_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..931ede568a3faa337637be06bccfd9ca136d8bc2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_tir.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_tir_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_tso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_tso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc4c0f1a3278259574fc84fd09be60422174b871 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_tso.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_tso_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_tum.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_tum.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c099dc6fd73bdad8c8e9e4d306cee8d9dec243fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_tum.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tum_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_tum_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..00501281a217c4d68834e8a1cd6dec9463e87268 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_twi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: twi_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_twi_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_tzm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_tzm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3915fa18f2c7963e0b1a0f4f10ca1da87f765141 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_tzm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tzm_Tfng +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_tzm_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_umb.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_umb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7f1cc79736bfba50fb8ab03c8749a77523feec4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_umb.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: umb_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_umb_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc2440248af154acd2aecdfc6d341230d4bfa67a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_wol.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_wol_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e075b84c190d949ad8e177b06445e0136f0445d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_xho.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_xho_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41ef062098ebc4b54b7aec5da59851d490924e6c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_yor_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7fc2f85efc9f9799446133ac107b6b0d66cfb38b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/sib_zul.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_zul_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_4/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib new file mode 100644 index 0000000000000000000000000000000000000000..812df7f614a9c8146b6da3137f4c2e97049b07f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib @@ -0,0 +1,43 @@ +tag: + - sib_tasks + - sib_prompt_5 + - afrobench_TC_tasks +dataset_path: Davlan/sib200 +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: category +doc_to_choice: + - "science/technology" + - "travel" + - "politics" + - "sports" + - "health" + - "entertainment" + - "geography" +should_decontaminate: true +doc_to_decontamination_query: text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_aeb.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_aeb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c88c0a28bdd40981ba847762e3cc08b36e66690 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_aeb.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: aeb_Arab +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Tunisian Arabic text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_aeb_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_afr.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_afr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d585478be65a678629dd3718c9e44f25a42b5e6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_afr.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Afrikaans text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_afr_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_aka.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_aka.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4644bfa3c9c923d544f86f13b273c5f754b236f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_aka.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: aka_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Akan text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_aka_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_amh.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2b5e6f9223f1b20d1d02b5f27635a0684388744 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_amh.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Amharic text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_amh_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_ary.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_ary.yaml new file mode 100644 index 0000000000000000000000000000000000000000..348c849d219b06501167c51182458f1946f51439 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_ary.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Moroccan Arabic text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_ary_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_arz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_arz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..100570428142a0f35ec728558251e96ec484ccb5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_arz.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Egyptian Arabic text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_arz_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_bam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_bam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bdc655003fc959e8219ae681b64b82dd137853d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_bam.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Bambara text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_bam_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_bem.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_bem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d42ea873b83e69d3d3d621a9ba1fafd7a88a4ab3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_bem.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Bemba text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_bem_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_cjk.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_cjk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9623b8c52bb39ab7dfa0f743d5482169a462b7ab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_cjk.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: cjk_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Chokwe text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_cjk_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_dik.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_dik.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83e76e963fe2962b472b8312d35a79c4e14d2b55 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_dik.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: dik_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Southwestern Dinka text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_dik_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_dyu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_dyu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ab215e89f959e6edf4bd07d1729f0424e85e0a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_dyu.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: dyu_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Dyula text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_dyu_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_eng.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a17a006d21d32ce9901010cbdcd94aad3af933f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_eng.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ English text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_eng_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_ewe.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..195876998160addea6184dbc4d3375192068aec5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_ewe.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Ewe text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_ewe_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_fon.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_fon.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61980b5110a424ed7391b29dbedf7f7828563f03 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_fon.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: fon_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Fon text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_fon_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_fra.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29573054bcfa08ddf225f925cb6131b6d4909163 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_fra.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ French text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_fra_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_fuv.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_fuv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b48f9f4e09e7042dfdec6ffa224300ee824b580 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_fuv.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Nigerian Fulfulde text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_fuv_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_gaz.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_gaz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37e2a4f97793217ecbd9f8c551d585901366cd34 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_gaz.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ West Central Oromo text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_gaz_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_hau.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24ce0970e9923725e2255abe09dc6e1629c9d23f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_hau.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Hausa text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_hau_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_ibo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a39ee75cb90a18ddb12bbedc54fd82a4c4c45ded --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_ibo.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Igbo text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_ibo_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kab.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kab.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d29da033388668302aca70c3511d52377e8797d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kab.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kab_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kabyle text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kab_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kam.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e55d1218586efb9f8d0b9bad3e9c2c76727a74b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kam.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kam_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kamba text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kam_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kbp.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kbp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..210baea8685a19c4e6bee6bcd141f2f0cb2a101a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kbp.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kbp_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kabiye text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kbp_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kea.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kea.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34a6813c8eff67f57adff7f43c982006b431ccf6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kea.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kabuverdianu text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kea_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kik.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kik.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55fdcb00e3f003422315e8de2ef64c8aa9e0abbc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kik.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kik_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kikuyu text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kik_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6567d52bf1f47beaba19141ab2f95b8168298290 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kin.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kinyarwanda text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kin_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kmb.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kmb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ae05cd06aef47942e20bbf72092cd23a5b4fb2f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kmb.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kmb_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kimbundu text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kmb_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_knc.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_knc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9870bd64740561b60f69af908c01ab221585d3fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_knc.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: knc_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Central Kanuri text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_knc_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kon.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kon.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afcab8b8dd34cfa63b1653223c05170789dddc10 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_kon.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kon_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kikongo text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kon_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_lin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c1611fd77106ae1eeef87ff5fcce60221ef8039 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_lin.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Lingala text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_lin_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_lua.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_lua.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3b2b9edcb68cf91b5c63273e07bedc936a7ffe1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_lua.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: lua_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Luba-Kasai text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_lua_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_lug.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8ca880ace46c5220acdcf2fb5d47bf37e2791aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_lug.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Luganda text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_lug_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_luo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_luo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b942d69d0cf5a367213aca5cc437b26015c83ae3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_luo.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Luo text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_luo_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_mos.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_mos.yaml new file mode 100644 index 0000000000000000000000000000000000000000..daccd62e9345db0c0625c7a26df993fe4f528411 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_mos.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: mos_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Mossi text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_mos_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_nso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_nso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09936e3c333c1cb2c285fc056d6da25246bcfefe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_nso.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Northern Sotho text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_nso_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_nus.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_nus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5f8e101910ba130d16708ce63388b4808286236 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_nus.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: nus_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Nuer text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_nus_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_nya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_nya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65737777ba6914aa1a735a260ee7ce7e3bfa9754 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_nya.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Nyanga text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_nya_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_plt.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_plt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24f6ea33e114f92a89b2b08581c3e2d93985362f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_plt.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Plateau Malagasy text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_plt_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_por.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_por.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d98ee118637f21a2ff1ffa30cd84099327965cbc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_por.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Portuguese text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_por_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_run.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_run.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01820da52cbd3b30dfe1c65a052b47ce1af0c7c0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_run.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: run_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Rundi text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_run_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_sag.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_sag.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fdabdcb63ca35cc8fd419558911099c7d8f14877 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_sag.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: sag_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Sango text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_sag_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_sna.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d66f53a7736d27346e47675426abfd4b63b6388 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_sna.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Shona text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_sna_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_som.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_som.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0c34f97d20b7c7e4b7be5b7225bf6a91baec3e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_som.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Somali text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_som_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_sot.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81ab5c3f7e66f457d59edef55eb79c693c18913d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_sot.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Southern Sotho text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_sot_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_ssw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_ssw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f662d2ab44cebe8ec184c7864207b9ceafa95f58 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_ssw.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Swazi text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_ssw_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_swa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee47ca51598f6455804e6e6cad3fb1ca1cacc4d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_swa.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Swahili text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_swa_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_taq.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_taq.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3fa1380df4b219fae78d9af94069a3becc256832 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_taq.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: taq_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Tamasheq text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_taq_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_tir.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_tir.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20ec0638837c561c45b0267d47a2a7481a3e9ec7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_tir.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Tigrinya text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_tir_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_tso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_tso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..44b3b867a796111bbcfe2d295ff5c04435878208 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_tso.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Tsonga text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_tso_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_tum.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_tum.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb15fb71e821e69e255b36183d2273a75292fd60 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_tum.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: tum_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Tumbuka text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_tum_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_twi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..44bca6194bc19417067ce10659958d0c5993ad87 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_twi.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: twi_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Twi text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_twi_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_tzm.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_tzm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d1af77d17ccc2e287c4e598b0094383ec5e4b01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_tzm.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: tzm_Tfng +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Tamazight text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_tzm_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_umb.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_umb.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a756680cfa29d6b4c83363192d734196989b2d45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_umb.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: umb_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Umbundu text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_umb_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_wol.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8062b55d71066a62df167f96c8e6a72b67e51b60 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_wol.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Wolof text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_wol_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_xho.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22c27b71a7e08c4f9c58039f5f52a10df324a878 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_xho.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Xhosa text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_xho_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_yor.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df51978255654580b37eca4e552e4561a6455e69 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_yor.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Yoruba text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_yor_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_zul.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03fb9af917049b8dac781d9aefac58ebd3fe4dba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/sib_zul.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Zulu text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_zul_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/prompt_5/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/sib.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/sib.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6935fee28978fd5f7efb02afd1a54dac363d111 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/sib.yaml @@ -0,0 +1,13 @@ +group: sib +task: + - sib_prompt_1 + - sib_prompt_2 + - sib_prompt_3 + - sib_prompt_4 + - sib_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d99649e343fa4c491c77cb3167c89cc09907f579 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/sib/utils.py @@ -0,0 +1,227 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_1": "Given the categories science/technology, travel, politics, sports, health, entertainment, or geography; what category does the text: '{{text}}' belong to: \n\n", + "prompt_2": f"Does this {lang} topic; " + "'{{text}}' belong to one of the following categories: science/technology, travel, politics, sports, health, entertainment, or geography? category only\n\n", + "prompt_3": f"You are an assistant able to classify topics in texts. \n\n" + f"Given the categories science/technology, travel, politics, sports, health, entertainment, or geography; what is " + f"the topic of the {lang} statement below? Return only the category. " + "\n\ntext: {{text}} \category:\n\n", + "prompt_4": "Label the following text as science/technology, travel, politics, sports, health, entertainment, or geography. Provide only the category as your " + "response. \n\ntext: {{text}} \category: \n\n", + "prompt_5": f"You are tasked with performing topic classification on the following {lang} text. " + f"For each input, classify the topic as science/technology, travel, politics, sports, health, entertainment, or geography. " + f"Use the following guidelines: \n\n " + f"science/technology: The text discusses scientific discoveries, technological advancements, or related topics. \n" + f"travel: The text describes travel experiences, destinations, or related topics. \n" + f"politics: The text covers political events, policies, or related topics. \n" + f"sports: The text talks about sports events, athletes, or related topics. \n" + f"health: The text addresses health issues, medical advancements, or related topics. \n" + f"entertainment: The text pertains to movies, music, celebrities, or related topics. \n" + f"geography: The text involves geographical information, locations, or related topics. \n\n" + f"If the text contains multiple topics, choose the dominant topic. " + f"For ambiguous or unclear topics, select the category that best reflects the overall content. " + "Please provide a single classification for each input.\n\ntext: {{text}} \category: \n\n", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "aeb": "Tunisian Arabic", + "afr": "Afrikaans", + "aka": "Akan", + "amh": "Amharic", + "ary": "Moroccan Arabic", + "arz": "Egyptian Arabic", + "bam": "Bambara", + "bem": "Bemba", + "cjk": "Chokwe", + "dik": "Southwestern Dinka", + "dyu": "Dyula", + "eng": "English", + "ewe": "Ewe", + "fon": "Fon", + "fra": "French", + "fuv": "Nigerian Fulfulde", + "gaz": "West Central Oromo", + "hau": "Hausa", + "ibo": "Igbo", + "kab": "Kabyle", + "kam": "Kamba", + "kmb": "Kimbundu", + "kbp": "Kabiye", + "kea": "Kabuverdianu", + "kik": "Kikuyu", + "kin": "Kinyarwanda", + "kon": "Kikongo", + "knc": "Central Kanuri", + "lua": "Luba-Kasai", + "lug": "Luganda", + "luo": "Luo", + "lin": "Lingala", + "mos": "Mossi", + "nus": "Nuer", + "nso": "Northern Sotho", + "nya": "Nyanga", + "plt": "Plateau Malagasy", + "por": "Portuguese", + "run": "Rundi", + "sag": "Sango", + "sna": "Shona", + "som": "Somali", + "sot": "Southern Sotho", + "ssw": "Swazi", + "swa": "Swahili", + "taq": "Tamasheq", + "tir": "Tigrinya", + "tum": "Tumbuka", + "tso": "Tsonga", + "twi": "Twi", + "tzm": "Tamazight", + "umb": "Umbundu", + "wol": "Wolof", + "xho": "Xhosa", + "yor": "Yoruba", + "zul": "Zulu", + } + + lang_2_dataset_lang_code = { + "aeb": "aeb_Arab", + "afr": "afr_Latn", + "aka": "aka_Latn", + "amh": "amh_Ethi", + "ary": "ary_Arab", + "arz": "arz_Arab", + "bam": "bam_Latn", + "bem": "bem_Latn", + "cjk": "cjk_Latn", + "dik": "dik_Latn", + "dyu": "dyu_Latn", + "eng": "eng_Latn", + "ewe": "ewe_Latn", + "fon": "fon_Latn", + "fra": "fra_Latn", + "fuv": "fuv_Latn", + "gaz": "gaz_Latn", + "hau": "hau_Latn", + "ibo": "ibo_Latn", + "kab": "kab_Latn", + "kam": "kam_Latn", + "kmb": "kmb_Latn", + "kbp": "kbp_Latn", + "kea": "kea_Latn", + "kik": "kik_Latn", + "kin": "kin_Latn", + "kon": "kon_Latn", + "knc": "knc_Latn", + "lua": "lua_Latn", + "lug": "lug_Latn", + "luo": "luo_Latn", + "lin": "lin_Latn", + "mos": "mos_Latn", + "nus": "nus_Latn", + "nso": "nso_Latn", + "nya": "nya_Latn", + "plt": "plt_Latn", + "por": "por_Latn", + "run": "run_Latn", + "sag": "sag_Latn", + "sna": "sna_Latn", + "som": "som_Latn", + "sot": "sot_Latn", + "ssw": "ssw_Latn", + "swa": "swh_Latn", + "taq": "taq_Latn", + "tir": "tir_Ethi", + "tum": "tum_Latn", + "tso": "tso_Latn", + "twi": "twi_Latn", + "tzm": "tzm_Tfng", + "umb": "umb_Latn", + "wol": "wol_Latn", + "xho": "xho_Latn", + "yor": "yor_Latn", + "zul": "zul_Latn", + } + + for lang in languages.keys(): + try: + file_name = f"sib_{lang}.yaml" + task_name = f"sib_{lang}_{mode}" + yaml_template = "sib" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang_2_dataset_lang_code[lang], + "doc_to_text": prompt_func(mode, languages[lang]), + } + file_path = os.path.join(output_dir, mode) + os.makedirs(file_path, exist_ok=True) + + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_3", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/README.md b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a0253f987e3723c309bcb5ce4c9a9ad2b3a166ec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/README.md @@ -0,0 +1,25 @@ +# + +## Paper +Title: `Uhura: A Benchmark for Evaluating Scientific Question Answering and Truthfulness in Low-Resource African Languages` + +Paper Link: https://arxiv.org/abs/2412.00948 + +## Abstract +>Evaluations of Large Language Models (LLMs) on knowledge-intensive tasks and factual accuracy often focus on high-resource languages primarily because datasets for low-resource languages (LRLs) are scarce. In this paper, we present Uhura -- a new benchmark that focuses on two tasks in six typologically-diverse African languages, created via human translation of existing English benchmarks. The first dataset, Uhura-ARC-Easy, is composed of multiple-choice science questions. The second, Uhura-TruthfulQA, is a safety benchmark testing the truthfulness of models on topics including health, law, finance, and politics. We highlight the challenges creating benchmarks with highly technical content for LRLs and outline mitigation strategies. Our evaluation reveals a significant performance gap between proprietary models such as GPT-4o and o1-preview, and Claude models, and open-source models like Meta's LLaMA and Google's Gemma. Additionally, all models perform better in English than in African languages. These results indicate that LMs struggle with answering scientific questions and are more prone to generating false claims in low-resource African languages. Our findings underscore the necessity for continuous improvement of multilingual LM capabilities in LRL settings to ensure safe and reliable use in real-world contexts. We open-source the Uhura Benchmark and Uhura Platform to foster further research and development in NLP for LRLs. + +HomePage: https://huggingface.co/datasets/masakhane/uhura-arc-easy + +### Citation + +``` +@misc{bayes2024uhurabenchmarkevaluatingscientific, + title={Uhura: A Benchmark for Evaluating Scientific Question Answering and Truthfulness in Low-Resource African Languages}, + author={Edward Bayes and Israel Abebe Azime and Jesujoba O. Alabi and Jonas Kgomo and Tyna Eloundou and Elizabeth Proehl and Kai Chen and Imaan Khadir and Naome A. Etori and Shamsuddeen Hassan Muhammad and Choice Mpanza and Igneciah Pocia Thete and Dietrich Klakow and David Ifeoluwa Adelani}, + year={2024}, + eprint={2412.00948}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2412.00948}, +} +``` diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy new file mode 100644 index 0000000000000000000000000000000000000000..a7e37181359d9021a3dbb669c42a0e80e0b36c8f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy @@ -0,0 +1,39 @@ +tag: + - uhura_arc_easy_tasks + - uhura_arc_easy_prompt_1 +task: null +dataset_path: masakhane/uhura-arc-easy +dataset_name: null +output_type: multiple_choice +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answerKey)}}" +doc_to_choice: + - A + - B + - C + - D +test_split: test +fewshot_split: validation +should_decontaminate: false +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_am.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_am.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f61efe4ea501ae6ee9c7553a05bdb7d8540c08f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_am.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: am_multiple_choice +doc_to_text: "You are a virtual assistant that answers multiple-choice questions with\ + \ the correct option only.\n\nQuestion: {{question}}\n\nChoices:\n\n{% for i in\ + \ range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n\ + {% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_am_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1b879e0221b72d2da47f4fe033f83f20526fef2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_en.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: en_multiple_choice +doc_to_text: "You are a virtual assistant that answers multiple-choice questions with\ + \ the correct option only.\n\nQuestion: {{question}}\n\nChoices:\n\n{% for i in\ + \ range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n\ + {% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_en_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_ha.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_ha.yaml new file mode 100644 index 0000000000000000000000000000000000000000..986ac5074660ef3c9756e1112e8aa5f34eafefe2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_ha.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ha_multiple_choice +doc_to_text: "You are a virtual assistant that answers multiple-choice questions with\ + \ the correct option only.\n\nQuestion: {{question}}\n\nChoices:\n\n{% for i in\ + \ range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n\ + {% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_ha_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_nso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_nso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ead6d97d67ec6f76e5378899e979413b6f6bb41b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_nso.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: nso_multiple_choice +doc_to_text: "You are a virtual assistant that answers multiple-choice questions with\ + \ the correct option only.\n\nQuestion: {{question}}\n\nChoices:\n\n{% for i in\ + \ range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n\ + {% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_nso_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_sw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e07bb234736d67e91cbcf798d6992fda2f438ec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_sw.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: sw_multiple_choice +doc_to_text: "You are a virtual assistant that answers multiple-choice questions with\ + \ the correct option only.\n\nQuestion: {{question}}\n\nChoices:\n\n{% for i in\ + \ range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n\ + {% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_sw_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_yo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_yo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f96113e4a5d3712fec89aab27fb309c0b85551b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_yo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: yo_multiple_choice +doc_to_text: "You are a virtual assistant that answers multiple-choice questions with\ + \ the correct option only.\n\nQuestion: {{question}}\n\nChoices:\n\n{% for i in\ + \ range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n\ + {% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_yo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_zu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_zu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41c965a071018685e0539ae8ee0f18389d4a0d01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_zu.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: zu_multiple_choice +doc_to_text: "You are a virtual assistant that answers multiple-choice questions with\ + \ the correct option only.\n\nQuestion: {{question}}\n\nChoices:\n\n{% for i in\ + \ range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n\ + {% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_zu_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy new file mode 100644 index 0000000000000000000000000000000000000000..295d9c8e907f841f74f8e9b7253d52c6eee2b224 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy @@ -0,0 +1,38 @@ +tag: + - uhura_arc_easy_tasks + - uhura_arc_easy_prompt_2 +dataset_path: masakhane/uhura-arc-easy +dataset_name: null +output_type: multiple_choice +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answerKey)}}" +doc_to_choice: + - A + - B + - C + - D +test_split: test +fewshot_split: validation +should_decontaminate: false +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_am.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_am.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2596bd487078859572e018d19f6f39c5e32f3dc5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_am.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: am_multiple_choice +doc_to_text: "Choose the correct option that answers the question below:\n\nQuestion:\ + \ {{question}}\n\nChoices:\n\n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i]\ + \ }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_am_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3edfc10ea1ada6252d938e57aed3a1f03ade802 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_en.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: en_multiple_choice +doc_to_text: "Choose the correct option that answers the question below:\n\nQuestion:\ + \ {{question}}\n\nChoices:\n\n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i]\ + \ }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_en_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_ha.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_ha.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d857b2e44c5fb9a7da2ac1e8dfbb426978799073 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_ha.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ha_multiple_choice +doc_to_text: "Choose the correct option that answers the question below:\n\nQuestion:\ + \ {{question}}\n\nChoices:\n\n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i]\ + \ }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_ha_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_nso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_nso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93fbfe587dc9dfb45c08aaaeb3c6c3528d766110 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_nso.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: nso_multiple_choice +doc_to_text: "Choose the correct option that answers the question below:\n\nQuestion:\ + \ {{question}}\n\nChoices:\n\n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i]\ + \ }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_nso_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_sw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5fc929f54de32f0010844d2ed816cd3b634184f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_sw.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sw_multiple_choice +doc_to_text: "Choose the correct option that answers the question below:\n\nQuestion:\ + \ {{question}}\n\nChoices:\n\n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i]\ + \ }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_sw_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_yo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_yo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..67b09752a4491d345d61bbbba5219cc1f5001544 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_yo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yo_multiple_choice +doc_to_text: "Choose the correct option that answers the question below:\n\nQuestion:\ + \ {{question}}\n\nChoices:\n\n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i]\ + \ }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_yo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_zu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_zu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b261b51fcc11770a22eaf0bd8285b2028d416b9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_zu.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: zu_multiple_choice +doc_to_text: "Choose the correct option that answers the question below:\n\nQuestion:\ + \ {{question}}\n\nChoices:\n\n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i]\ + \ }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_zu_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy new file mode 100644 index 0000000000000000000000000000000000000000..23e2c37396c75ce09fcd608e69d5ce42173df1ca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy @@ -0,0 +1,38 @@ +tag: + - uhura_arc_easy_tasks + - uhura_arc_easy_prompt_3 +dataset_path: masakhane/uhura-arc-easy +dataset_name: null +output_type: multiple_choice +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answerKey)}}" +doc_to_choice: + - A + - B + - C + - D +test_split: test +fewshot_split: validation +should_decontaminate: false +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_am.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_am.yaml new file mode 100644 index 0000000000000000000000000000000000000000..42716a7cdc9b1ac266efbec32ebe4bebe6bf578e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_am.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: am_multiple_choice +doc_to_text: "Answer the following multiple-choice question by picking 'A', 'B', 'C',\ + \ or 'D'.\n\nQuestion: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_am_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a89312e09a10b9db5a2a9a2a0914980c9ef686a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_en.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: en_multiple_choice +doc_to_text: "Answer the following multiple-choice question by picking 'A', 'B', 'C',\ + \ or 'D'.\n\nQuestion: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_en_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_ha.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_ha.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de511a8af7dc52180b56d52a1c7d57955cbd6eb4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_ha.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ha_multiple_choice +doc_to_text: "Answer the following multiple-choice question by picking 'A', 'B', 'C',\ + \ or 'D'.\n\nQuestion: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_ha_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_nso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_nso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..358d084cea2131f7e94f82ec733234371f2b8446 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_nso.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: nso_multiple_choice +doc_to_text: "Answer the following multiple-choice question by picking 'A', 'B', 'C',\ + \ or 'D'.\n\nQuestion: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_nso_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_sw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4a8785d622caed597dbb0eefbd4290f2636f866 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_sw.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sw_multiple_choice +doc_to_text: "Answer the following multiple-choice question by picking 'A', 'B', 'C',\ + \ or 'D'.\n\nQuestion: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_sw_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_yo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_yo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9416362827513ac4b95cf843a83ec0d0efbc45e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_yo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yo_multiple_choice +doc_to_text: "Answer the following multiple-choice question by picking 'A', 'B', 'C',\ + \ or 'D'.\n\nQuestion: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_yo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_zu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_zu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a44b8c0e6ebae36db00a6847aacb3263c84fb7b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_zu.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: zu_multiple_choice +doc_to_text: "Answer the following multiple-choice question by picking 'A', 'B', 'C',\ + \ or 'D'.\n\nQuestion: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_zu_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy new file mode 100644 index 0000000000000000000000000000000000000000..e697f4c7363aee6c39b0d927ba9d1b575f4063d5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy @@ -0,0 +1,38 @@ +tag: + - uhura_arc_easy_tasks + - uhura_arc_easy_prompt_4 +dataset_path: masakhane/uhura-arc-easy +dataset_name: null +output_type: multiple_choice +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answerKey)}}" +doc_to_choice: + - A + - B + - C + - D +test_split: test +fewshot_split: validation +should_decontaminate: false +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_am.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_am.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4eaa02f59b217a8ce13f41fca67f9491aab917aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_am.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: am_multiple_choice +doc_to_text: "Question: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_am_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..461e6f9e7516e2382380e391f3d2d713bf494ef9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_en.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: en_multiple_choice +doc_to_text: "Question: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_en_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_ha.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_ha.yaml new file mode 100644 index 0000000000000000000000000000000000000000..435ea73bc7639669ac53445f0ea9adf72edbd347 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_ha.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ha_multiple_choice +doc_to_text: "Question: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_ha_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_nso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_nso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09112d5af3adaf7251a39032aabf60af73779088 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_nso.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_multiple_choice +doc_to_text: "Question: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_nso_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_sw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..264770eeda75acdf9088ac24e24645a9e5638c25 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_sw.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sw_multiple_choice +doc_to_text: "Question: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_sw_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_yo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_yo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10af53de81d71c82f02f1da80bdc9b5dc114bfed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_yo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yo_multiple_choice +doc_to_text: "Question: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_yo_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_zu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_zu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..239b1648a6e2c5d431a83ef0a94c16c6db90cc1b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_zu.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zu_multiple_choice +doc_to_text: "Question: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_zu_prompt_4 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy new file mode 100644 index 0000000000000000000000000000000000000000..3f5ac554027a87a6fe5eeda14887a46f5af5ef2f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy @@ -0,0 +1,38 @@ +tag: + - uhura_arc_easy_tasks + - uhura_arc_easy_prompt_5 +dataset_path: masakhane/uhura-arc-easy +dataset_name: null +output_type: multiple_choice +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answerKey)}}" +doc_to_choice: + - A + - B + - C + - D +test_split: test +fewshot_split: validation +should_decontaminate: false +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_am.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_am.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7f0231017eb5553893708f939b5fb23d0f60e1e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_am.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: am_multiple_choice +doc_to_text: "Which of the following options answers this question: {{question}}\n\ + \n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i]\ + \ }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_am_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_en.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5aea6abac239580735b805c2582be3976a9986d4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_en.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: en_multiple_choice +doc_to_text: "Which of the following options answers this question: {{question}}\n\ + \n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i]\ + \ }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_en_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_ha.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_ha.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6293bda284e9ca224c3be7a65e1686f3c97210d8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_ha.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ha_multiple_choice +doc_to_text: "Which of the following options answers this question: {{question}}\n\ + \n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i]\ + \ }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_ha_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_nso.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_nso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80aff7064e48444477970baf5ccedf930a560a34 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_nso.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: nso_multiple_choice +doc_to_text: "Which of the following options answers this question: {{question}}\n\ + \n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i]\ + \ }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_nso_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_sw.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_sw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5bc7d2e5600b46a8660c83e76c69b9a24d0f398 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_sw.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sw_multiple_choice +doc_to_text: "Which of the following options answers this question: {{question}}\n\ + \n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i]\ + \ }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_sw_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_yo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_yo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a267e987218945fc09217266defb4b0775fd777f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_yo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yo_multiple_choice +doc_to_text: "Which of the following options answers this question: {{question}}\n\ + \n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i]\ + \ }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_yo_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_zu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_zu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69ce4a396af9f0bd6c96071319ef51ac3c1a81cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_zu.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: zu_multiple_choice +doc_to_text: "Which of the following options answers this question: {{question}}\n\ + \n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i]\ + \ }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_zu_prompt_5 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/uhura.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/uhura.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2e2fea5fb49838103490f6f16321da45022cc7c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/uhura.yaml @@ -0,0 +1,13 @@ +group: uhura_arc_easy +task: + - uhura_arc_easy_prompt_1 + - uhura_arc_easy_prompt_2 + - uhura_arc_easy_prompt_3 + - uhura_arc_easy_prompt_4 + - uhura_arc_easy_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1216618cbff12e6f4a21ff532d7da16abbef1bde --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/uhura-arc-easy/utils.py @@ -0,0 +1,129 @@ +import argparse +import os + +import pycountry +import yaml + + +def get_language_from_code(code: str) -> str: + language_tuple = pycountry.languages.get(**{f"alpha_{len(code)}": code}) + return language_tuple.name + + +def prompt_func(mode): + prompt_map = { + "prompt_1": "You are a virtual assistant that answers multiple-choice questions with the correct option only.\n\n" + "Question: {{question}}\n\n" + "Choices:\n\n" + "{% for i in range(choices['text']|length) %}" + "\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n" + "{% endfor %}\n" + "Answer: ", + "prompt_2": "Choose the correct option that answers the question below:\n\n" + "Question: {{question}}\n\n" + "Choices:\n\n" + "{% for i in range(choices['text']|length) %}" + "\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n" + "{% endfor %}\n" + "Answer: ", + "prompt_3": "Answer the following multiple-choice question by picking 'A', 'B', 'C', or 'D'.\n\n" + "Question: {{question}}\n\n" + "Options:\n\n" + "{% for i in range(choices['text']|length) %}" + "\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n" + "{% endfor %}\n" + "Answer: ", + "prompt_4": "Question: {{question}}\n\n" + "Options:\n\n" + "{% for i in range(choices['text']|length) %}" + "\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n" + "{% endfor %}\n" + "Answer: ", + "prompt_5": "Which of the following options answers this question: {{question}}\n\n" + "{% for i in range(choices['text']|length) %}" + "\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n" + "{% endfor %}\n" + "Answer: ", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = {"am", "en", "ha", "nso", "sw", "yo", "zu"} + + for lang in languages: + try: + file_name = f"uhura-arc-easy_{lang}.yaml" + task_name = f"uhura-arc-easy_{lang}_{mode}" + yaml_template = "uhura-arc-easy" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": f"{lang}_multiple_choice{'_unmatched' if lang == 'nso' else ''}", + "doc_to_text": prompt_func(mode), + } + if lang in ("nso", "zu"): + yaml_details["fewshot_split"] = "train" + + file_path = os.path.join(output_dir, mode) + os.makedirs(file_path, exist_ok=True) + + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + + PROMPT_CHOICES = ["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"] + parser.add_argument( + "--mode", + nargs="*", + default=PROMPT_CHOICES, + choices=PROMPT_CHOICES, + help="Prompt number(s)", + ) + args = parser.parse_args() + + for mode in args.mode: + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=mode) + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/README.md b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d9a47076e564de38fb4a7eb2cbd1df8a3b0290d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/README.md @@ -0,0 +1,34 @@ +# + +## Paper +Title: `XL-Sum: Large-Scale Multilingual Abstractive Summarization for 44 Languages` + +Paper Link: https://aclanthology.org/2021.findings-acl.413/ + +## Abstract +>Contemporary works on abstractive text summarization have focused primarily on high-resource languages like English, mostly due to the limited availability of datasets for low/mid-resource ones. In this work, we present XL-Sum, a comprehensive and diverse dataset comprising 1 million professionally annotated article-summary pairs from BBC, extracted using a set of carefully designed heuristics. The dataset covers 44 languages ranging from low to high-resource, for many of which no public dataset is currently available. XL-Sum is highly abstractive, concise, and of high quality, as indicated by human and intrinsic evaluation. We fine-tune mT5, a state-of-the-art pretrained multilingual model, with XL-Sum and experiment on multilingual and low-resource summarization tasks. XL-Sum induces competitive results compared to the ones obtained using similar monolingual datasets: we show higher than 11 ROUGE-2 scores on 10 languages we benchmark on, with some of them exceeding 15, as obtained by multilingual training. Additionally, training on low-resource languages individually also provides competitive performance. To the best of our knowledge, XL-Sum is the largest abstractive summarization dataset in terms of the number of samples collected from a single source and the number of languages covered. We are releasing our dataset and models to encourage future research on multilingual abstractive summarization. + +HomePage: https://github.com/csebuetnlp/xl-sum + +### Citation + +``` +@inproceedings{hasan-etal-2021-xl, + title = "{XL}-Sum: Large-Scale Multilingual Abstractive Summarization for 44 Languages", + author = "Hasan, Tahmid and + Bhattacharjee, Abhik and + Islam, Md. Saiful and + Mubasshir, Kazi and + Li, Yuan-Fang and + Kang, Yong-Bin and + Rahman, M. Sohel and + Shahriyar, Rifat", + booktitle = "Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021", + month = aug, + year = "2021", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2021.findings-acl.413", + pages = "4693--4703", +} +``` diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..85db4d4f4cef061e526c970ece194317e576de06 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/utils.py @@ -0,0 +1,18 @@ +import evaluate + + +def rougeL(items): + """ + # passthrough for efficiency + """ + return items + + +def rougeL_agg(items): + """ + Higher is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + rouge_scorer = evaluate.load("rouge") + return rouge_scorer.compute(predictions=preds, references=refs)["rougeL"] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum new file mode 100644 index 0000000000000000000000000000000000000000..f6b0421edd9ba2b3f1c2eac1dbfaf6f51e5cfba5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum @@ -0,0 +1,22 @@ +tag: + - xlsum_tasks + - xlsum_prompt_1 +task: null +dataset_path: csebuetnlp/xlsum +dataset_name: null +dataset_kwargs: + trust_remote_code: true +output_type: generate_until +generation_kwargs: + until: + - "" +validation_split: validation +fewshot_split: validation +test_split: test +should_decontaminate: false +metric_list: + - metric: !function utils.rougeL + higher_is_better: true + aggregation: !function utils.rougeL_agg +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_amharic.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_amharic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ab68805aa658c3c15d8367f48115f40e2581aac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_amharic.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: amharic +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Amharic. Ensure that you + provide the summary in Amharic and nothing else. + + Document in Amharic: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_amharic_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_arabic.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_arabic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af7df7d90f01b274c1d54076256d7e3a510627b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_arabic.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: arabic +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Arabic. Ensure that you + provide the summary in Arabic and nothing else. + + Document in Arabic: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_arabic_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_hausa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_hausa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37f6b3e518835365e7b59fb550c15e286c85f63a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_hausa.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: hausa +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Hausa. Ensure that you + provide the summary in Hausa and nothing else. + + Document in Hausa: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_hausa_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_igbo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_igbo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04644b5d7bdd8595c5beb02240fe521162dcf3fd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_igbo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: igbo +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Igbo. Ensure that you provide + the summary in Igbo and nothing else. + + Document in Igbo: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_igbo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_kirundi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_kirundi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c434296f5598cb995c40568ab69141f29571d57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_kirundi.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: kirundi +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Kirundi. Ensure that you + provide the summary in Kirundi and nothing else. + + Document in Kirundi: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_kirundi_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_oromo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_oromo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78fb14eca4344c17ed3300954193764568be40d4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_oromo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: oromo +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Oromo. Ensure that you + provide the summary in Oromo and nothing else. + + Document in Oromo: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_oromo_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_pidgin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_pidgin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68f2c17f560ee888ea1ee958c9ba2392d6f47dfc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_pidgin.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: pidgin +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Nigerian pidgin. Ensure + that you provide the summary in Nigerian pidgin and nothing else. + + Document in Nigerian pidgin: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_pidgin_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_somali.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_somali.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d699dc1905796945e89f3659060202f7314ed776 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_somali.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: somali +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Somali. Ensure that you + provide the summary in Somali and nothing else. + + Document in Somali: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_somali_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_swahili.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_swahili.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a951c11b8c7ee59f1dfcd3eb44eaada2bd0652a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_swahili.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: swahili +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Swahili. Ensure that you + provide the summary in Swahili and nothing else. + + Document in Swahili: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_swahili_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_telugu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_telugu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82a60171a5e2a42e1eb5d43aaf8a034e77b4a798 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_telugu.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: telugu +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Telugu. Ensure that you + provide the summary in Telugu and nothing else. + + Document in Telugu: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_telugu_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_tigrinya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_tigrinya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31630982a134b934311c00164c42dc9fabf22cc7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_tigrinya.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: tigrinya +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Tigrinya. Ensure that you + provide the summary in Tigrinya and nothing else. + + Document in Tigrinya: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_tigrinya_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_yoruba.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_yoruba.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c14a9113e293c057d028e27cd09ed1f6812c1e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_yoruba.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: yoruba +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Yoruba. Ensure that you + provide the summary in Yoruba and nothing else. + + Document in Yoruba: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_yoruba_prompt_1 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..85db4d4f4cef061e526c970ece194317e576de06 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/utils.py @@ -0,0 +1,18 @@ +import evaluate + + +def rougeL(items): + """ + # passthrough for efficiency + """ + return items + + +def rougeL_agg(items): + """ + Higher is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + rouge_scorer = evaluate.load("rouge") + return rouge_scorer.compute(predictions=preds, references=refs)["rougeL"] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum new file mode 100644 index 0000000000000000000000000000000000000000..e572c00c6ae1c0f8f84f1030c5903325ca1f0ae4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum @@ -0,0 +1,22 @@ +tag: + - xlsum_tasks + - xlsum_prompt_2 +task: null +dataset_path: csebuetnlp/xlsum +dataset_name: null +dataset_kwargs: + trust_remote_code: true +output_type: generate_until +generation_kwargs: + until: + - "" +validation_split: validation +fewshot_split: validation +test_split: test +should_decontaminate: false +metric_list: + - metric: !function utils.rougeL + higher_is_better: true + aggregation: !function utils.rougeL_agg +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_amharic.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_amharic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f2275c657b54df708f62526fdc12b0381f197eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_amharic.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: amharic +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_amharic_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_arabic.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_arabic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4f772c31610175417ee97105ae4d99f526f0c41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_arabic.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: arabic +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_arabic_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_hausa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_hausa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7485672cb48c86d1df444391064b04754836fa77 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_hausa.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: hausa +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_hausa_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_igbo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_igbo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2cf7fafe394e049aaa3382068d6da8cac70cf705 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_igbo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: igbo +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_igbo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_kirundi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_kirundi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63021d7d4f9ff0d252c82665fd8262bd6bb5c327 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_kirundi.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: kirundi +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_kirundi_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_oromo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_oromo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b637b10d0428d614fcd4c06bdb1fb2383057ef77 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_oromo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: oromo +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_oromo_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_pidgin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_pidgin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c13d93d5c1ff95ac76c1b87f4c301c97a771f52 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_pidgin.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: pidgin +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_pidgin_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_somali.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_somali.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7245ddc193a133701fd8f71cd6b52cd34899594 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_somali.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: somali +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_somali_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_swahili.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_swahili.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65f176fba40e37003a9cfd8813957760cdac2aa1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_swahili.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: swahili +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_swahili_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_telugu.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_telugu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ecbdde5c90806b2684fce1373c3fad94ef5c65e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_telugu.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: telugu +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_telugu_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_tigrinya.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_tigrinya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d46e2fb573f5cdd3bb9459c7eb5b95150cae5ec8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_tigrinya.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: tigrinya +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_tigrinya_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_yoruba.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_yoruba.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ea0ef503444a8237ce5fd693f6ebec082a8a6cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_yoruba.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: yoruba +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_yoruba_prompt_2 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..85db4d4f4cef061e526c970ece194317e576de06 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/utils.py @@ -0,0 +1,18 @@ +import evaluate + + +def rougeL(items): + """ + # passthrough for efficiency + """ + return items + + +def rougeL_agg(items): + """ + Higher is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + rouge_scorer = evaluate.load("rouge") + return rouge_scorer.compute(predictions=preds, references=refs)["rougeL"] diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum new file mode 100644 index 0000000000000000000000000000000000000000..08842ef8eb627dfb12387ae7ef2e232d2f3c40d3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum @@ -0,0 +1,22 @@ +tag: + - xlsum_tasks + - xlsum_prompt_3 +task: null +dataset_path: csebuetnlp/xlsum +dataset_name: null +dataset_kwargs: + trust_remote_code: true +output_type: generate_until +generation_kwargs: + until: + - "" +validation_split: validation +fewshot_split: validation +test_split: test +should_decontaminate: false +metric_list: + - metric: !function utils.rougeL + higher_is_better: true + aggregation: !function utils.rougeL_agg +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_amharic.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_amharic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fc85e7ceb43f474754081088a77b3979b785334 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_amharic.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: amharic +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Amharic. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_amharic_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_arabic.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_arabic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4f2b1f5c09bcf2dc76fb4807e09a1cc52b6ce82 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_arabic.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: arabic +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Arabic. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_arabic_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_hausa.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_hausa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1a0603749b196a9cda3f995fb16ea2814513142 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_hausa.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: hausa +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Hausa. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_hausa_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_igbo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_igbo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b23f8f395679740acd523757311a7803407c3cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_igbo.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: igbo +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Igbo. Your main goal is to ensure summaries are concise and + informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_igbo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_kirundi.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_kirundi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f40b2a7ff68847a5bae0649451460d22f24ae2a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_kirundi.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kirundi +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Kirundi. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_kirundi_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_oromo.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_oromo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbc912851b05e78829a2835e14e28831469d302d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_oromo.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: oromo +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Oromo. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_oromo_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_pidgin.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_pidgin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8149e441e9869a94312fe54001cd93fd6d720eaa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_pidgin.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: pidgin +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Nigerian pidgin. Your main goal is to ensure summaries are + concise and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_pidgin_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_somali.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_somali.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2936da11bdca6fb8fd9ba6f288968ee0c1843a4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_somali.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: somali +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Somali. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_somali_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_yoruba.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_yoruba.yaml new file mode 100644 index 0000000000000000000000000000000000000000..353be14cda6713964f73436b2085d6ae63fcdc57 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_yoruba.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: yoruba +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Yoruba. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_yoruba_prompt_3 diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/utils.py b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8df1e12e8b4aa683bd71c2fb23d90ff7667de5b2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/utils.py @@ -0,0 +1,118 @@ +import argparse +import os + +import yaml + + +def prompt_func(mode, lang): + if lang == "pidgin": + lang = "Nigerian Pidgin" + + prompt_map = { + "prompt_1": f"Provide a summary of the document written in {lang.capitalize()}. Ensure that you provide the summary in {lang.capitalize()} and nothing else.\n" + f"Document in {lang.capitalize()}: " + r"{{'text'}}\n" + "Summary: ", + "prompt_2": "Summarize the document below in triple backticks and return only the summary and nothing else.\n" + + r"```{{'text'}}```\n", + "prompt_3": f"You are an advanced Summarizer, a specialized assistant designed to summarize documents in {lang.capitalize()}. " + f"Your main goal is to ensure summaries are concise and informative. Ensure you return the summary only and nothing else.\n" + f"Document: " + r"{{'text'}}\n" + "Summary: ", + "prompt_4": f"Summarize this {lang.capitalize()} document:\n" + r"{{'text'}}\n" + "Summary: ", + "prompt_5": f"{lang.capitalize()} document: " + r"{{'text'}}\n" + "Summary: ", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + XLSUM_LANGUAGES = ( + "amharic", + "arabic", + "hausa", + "igbo", + "kirundi", + "oromo", + "pidgin", + "somali", + "swahili", + "telugu", + "tigrinya", + "yoruba", + ) + + for lang in XLSUM_LANGUAGES: + try: + file_name = f"xlsum_{lang}.yaml" + task_name = f"xlsum_{lang}_{mode}" + yaml_template = "xlsum" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + "doc_to_text": prompt_func(mode, lang), + "doc_to_target": "{{summary}}", + } + file_path = os.path.join(output_dir, mode) + os.makedirs(file_path, exist_ok=True) + + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + + PROMPT_CHOICES = ["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"] + parser.add_argument( + "--mode", + nargs="*", + default=PROMPT_CHOICES, + choices=PROMPT_CHOICES, + help="Prompt number(s)", + ) + args = parser.parse_args() + + for mode in args.mode: + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=mode) + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/xlsum.yaml b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/xlsum.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d87717597c59eb333f712d69eb854e971146915 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/afrobench/xlsum/xlsum.yaml @@ -0,0 +1,11 @@ +group: xlum +task: + - xlsum_prompt_1 + - xlsum_prompt_2 + - xlsum_prompt_3 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 2 diff --git a/lm-evaluation-harness/lm_eval/tasks/agieval/README.md b/lm-evaluation-harness/lm_eval/tasks/agieval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..53a9df036d6c0a4dcc2b310ac324f1bf7b0f60dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/agieval/README.md @@ -0,0 +1,118 @@ +# AGIEval + +### Paper + +Title: AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models + +Abstract: https://arxiv.org/abs/2304.06364.pdf + +AGIEval is a human-centric benchmark specifically designed to evaluate the general abilities of foundation models in tasks pertinent to human cognition and problem-solving. +This benchmark is derived from 20 official, public, and high-standard admission and qualification exams intended for general human test-takers, such as general college admission tests (e.g., Chinese College Entrance Exam (Gaokao) and American SAT), law school admission tests, math competitions, lawyer qualification tests, and national civil service exams. + +Homepage: https://github.com/ruixiangcui/AGIEval + +### Citation + +``` +@misc{zhong2023agieval, + title={AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models}, + author={Wanjun Zhong and Ruixiang Cui and Yiduo Guo and Yaobo Liang and Shuai Lu and Yanlin Wang and Amin Saied and Weizhu Chen and Nan Duan}, + year={2023}, + eprint={2304.06364}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +``` + +Please make sure to cite all the individual datasets in your paper when you use them. We provide the relevant citation information below: + +``` +@inproceedings{ling-etal-2017-program, + title = "Program Induction by Rationale Generation: Learning to Solve and Explain Algebraic Word Problems", + author = "Ling, Wang and + Yogatama, Dani and + Dyer, Chris and + Blunsom, Phil", + booktitle = "Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = jul, + year = "2017", + address = "Vancouver, Canada", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/P17-1015", + doi = "10.18653/v1/P17-1015", + pages = "158--167", + abstract = "Solving algebraic word problems requires executing a series of arithmetic operations{---}a program{---}to obtain a final answer. However, since programs can be arbitrarily complicated, inducing them directly from question-answer pairs is a formidable challenge. To make this task more feasible, we solve these problems by generating answer rationales, sequences of natural language and human-readable mathematical expressions that derive the final answer through a series of small steps. Although rationales do not explicitly specify programs, they provide a scaffolding for their structure via intermediate milestones. To evaluate our approach, we have created a new 100,000-sample dataset of questions, answers and rationales. Experimental results show that indirect supervision of program learning via answer rationales is a promising strategy for inducing arithmetic programs.", +} + +@inproceedings{hendrycksmath2021, + title={Measuring Mathematical Problem Solving With the MATH Dataset}, + author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt}, + journal={NeurIPS}, + year={2021} +} + +@inproceedings{Liu2020LogiQAAC, + title={LogiQA: A Challenge Dataset for Machine Reading Comprehension with Logical Reasoning}, + author={Jian Liu and Leyang Cui and Hanmeng Liu and Dandan Huang and Yile Wang and Yue Zhang}, + booktitle={International Joint Conference on Artificial Intelligence}, + year={2020} +} + +@inproceedings{zhong2019jec, + title={JEC-QA: A Legal-Domain Question Answering Dataset}, + author={Zhong, Haoxi and Xiao, Chaojun and Tu, Cunchao and Zhang, Tianyang and Liu, Zhiyuan and Sun, Maosong}, + booktitle={Proceedings of AAAI}, + year={2020}, +} + +@article{Wang2021FromLT, + title={From LSAT: The Progress and Challenges of Complex Reasoning}, + author={Siyuan Wang and Zhongkun Liu and Wanjun Zhong and Ming Zhou and Zhongyu Wei and Zhumin Chen and Nan Duan}, + journal={IEEE/ACM Transactions on Audio, Speech, and Language Processing}, + year={2021}, + volume={30}, + pages={2201-2216} +} +``` + +### Groups, Tags, and Tasks + +#### Groups + +- `agieval`: Evaluates all tasks listed below. + +- `agieval_en`: Evaluates all English subtasks: `agieval_aqua_rat`, `agieval_gaokao_english`, `agieval_logiqa_en`, `agieval_lsat_*`, `agieval_sat_*`, `agieval_math` + +- `agieval_cn`: Evaluates all Chinese subtasks: +`agieval_gaokao_biology`, `agieval_gaokao_chemistry`, `agieval_gaokao_chinese`, `agieval_gaokao_geography`, +`agieval_gaokao_history`, `agieval_gaokao_mathqa`, `agieval_gaokao_mathcloze`, `agieval_gaokao_physics`, `agieval_jec_qa_ca`, `agieval_jec_qa_kd`, `agieval_logiqa_zh` + +- `agieval_nous`: Evaluates a specific subset of AGIEval tasks (multiple-choice and english-only), namely those in https://github.com/teknium1/LLM-Benchmark-Logs/blob/main/benchmark-logs/Mistral-7B-Base.md + +#### Tags + +None. + +#### Tasks + +- `agieval_aqua_rat` +- `agieval_gaokao_biology` +- `agieval_gaokao_chemistry` +- `agieval_gaokao_chinese` +- `agieval_gaokao_english` +- `agieval_gaokao_geography` +- `agieval_gaokao_history` +- `agieval_gaokao_mathqa` +- `agieval_gaokao_mathcloze` +- `agieval_gaokao_physics` +- `agieval_jec_qa_ca` +- `agieval_jec_qa_kd` +- `agieval_logiqa_en` +- `agieval_logiqa_zh` +- `agieval_lsat_ar` +- `agieval_lsat_lr` +- `agieval_lsat_rc` +- `agieval_sat_en` +- `agieval_sat_en_without_passage` +- `agieval_sat_math` +- `agieval_math` diff --git a/lm-evaluation-harness/lm_eval/tasks/agieval/agieval.yaml b/lm-evaluation-harness/lm_eval/tasks/agieval/agieval.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d086af83579ec2daf826a55f7dd82cf2e1f82a96 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/agieval/agieval.yaml @@ -0,0 +1,29 @@ +group: agieval +task: + - agieval_gaokao_biology + - agieval_gaokao_chemistry + - agieval_gaokao_chinese + - agieval_gaokao_geography + - agieval_gaokao_history + - agieval_gaokao_mathcloze + - agieval_gaokao_mathqa + - agieval_gaokao_physics + - agieval_jec_qa_ca + - agieval_jec_qa_kd + - agieval_logiqa_zh + - agieval_aqua_rat + - agieval_gaokao_english + - agieval_logiqa_en + - agieval_lsat_ar + - agieval_lsat_lr + - agieval_lsat_rc + - agieval_math + - agieval_sat_en_without_passage + - agieval_sat_en + - agieval_sat_math +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/agieval/agieval_cn.yaml b/lm-evaluation-harness/lm_eval/tasks/agieval/agieval_cn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e8ca2fdedaabe768fe731bbf2dbbea3ef117448 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/agieval/agieval_cn.yaml @@ -0,0 +1,19 @@ +group: agieval_cn +task: + - agieval_gaokao_biology + - agieval_gaokao_chemistry + - agieval_gaokao_chinese + - agieval_gaokao_geography + - agieval_gaokao_history + - agieval_gaokao_mathcloze + - agieval_gaokao_mathqa + - agieval_gaokao_physics + - agieval_jec_qa_ca + - agieval_jec_qa_kd + - agieval_logiqa_zh +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/agieval/agieval_en.yaml b/lm-evaluation-harness/lm_eval/tasks/agieval/agieval_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a873d66d3a4e98fc2ce2df26e53f20a599bc4e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/agieval/agieval_en.yaml @@ -0,0 +1,18 @@ +group: agieval_en +task: + - agieval_aqua_rat + - agieval_gaokao_english # categorizing as EN because the AGIEval codebase lists this as in `english_qa_tasks` + - agieval_logiqa_en + - agieval_lsat_ar + - agieval_lsat_lr + - agieval_lsat_rc + - agieval_math + - agieval_sat_en_without_passage + - agieval_sat_en + - agieval_sat_math +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/agieval/agieval_nous.yaml b/lm-evaluation-harness/lm_eval/tasks/agieval/agieval_nous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa2a874892e77aaa0216feac4f2f6353b3302a93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/agieval/agieval_nous.yaml @@ -0,0 +1,16 @@ +group: agieval_nous +task: + - agieval_aqua_rat + - agieval_logiqa_en + - agieval_lsat_ar + - agieval_lsat_lr + - agieval_lsat_rc + - agieval_sat_en_without_passage + - agieval_sat_en + - agieval_sat_math +aggregate_metric_list: + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/agieval/aqua-rat.yaml b/lm-evaluation-harness/lm_eval/tasks/agieval/aqua-rat.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5a3a3e86f6c5448000df38a146a95518691b934 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/agieval/aqua-rat.yaml @@ -0,0 +1,20 @@ +task: agieval_aqua_rat +dataset_path: hails/agieval-aqua-rat +dataset_name: null +output_type: multiple_choice +training_split: null +validation_split: null +test_split: test +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "{{choices}}" +process_results: !function utils.process_results_mcqa +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/agieval/gaokao-biology.yaml b/lm-evaluation-harness/lm_eval/tasks/agieval/gaokao-biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8b9eca8397965a0bf3c7152fbd237236b0f37f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/agieval/gaokao-biology.yaml @@ -0,0 +1,3 @@ +include: aqua-rat.yaml +task: agieval_gaokao_biology +dataset_path: hails/agieval-gaokao-biology diff --git a/lm-evaluation-harness/lm_eval/tasks/agieval/gaokao-chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/agieval/gaokao-chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4ba01a5274376bab68fb8a56bd25a6e81d1edfb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/agieval/gaokao-chemistry.yaml @@ -0,0 +1,3 @@ +include: aqua-rat.yaml +task: agieval_gaokao_chemistry +dataset_path: hails/agieval-gaokao-chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/agieval/gaokao-chinese.yaml b/lm-evaluation-harness/lm_eval/tasks/agieval/gaokao-chinese.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d58b5bc495917482ef69f04604b7f78f91339f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/agieval/gaokao-chinese.yaml @@ -0,0 +1,3 @@ +include: aqua-rat.yaml +task: agieval_gaokao_chinese +dataset_path: hails/agieval-gaokao-chinese diff --git a/lm-evaluation-harness/lm_eval/tasks/agieval/gaokao-english.yaml b/lm-evaluation-harness/lm_eval/tasks/agieval/gaokao-english.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12ea66787acfa60eefc5a49936c6484b08c8fda0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/agieval/gaokao-english.yaml @@ -0,0 +1,3 @@ +include: aqua-rat.yaml +task: agieval_gaokao_english +dataset_path: hails/agieval-gaokao-english diff --git a/lm-evaluation-harness/lm_eval/tasks/agieval/gaokao-geography.yaml b/lm-evaluation-harness/lm_eval/tasks/agieval/gaokao-geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6dbce6f4873e272f9c28f49b0061857060df2e97 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/agieval/gaokao-geography.yaml @@ -0,0 +1,3 @@ +include: aqua-rat.yaml +task: agieval_gaokao_geography +dataset_path: hails/agieval-gaokao-geography