Add files using upload-large-folder tool

lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_zul.yaml

@@ -0,0 +1,8 @@
+# Generated by utils.py
+dataset_name: zul
+doc_to_text: "Given the following premise and hypothesis in Zulu, identify if the\
+  \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\
+  \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\
+  \ \nHypothesis: {{hypothesis}}"
+include: afrixnli_translate_yaml
+task: afrixnli_translate_zul_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_yor.yaml

@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: yor
+doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\
+  \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}"
+include: afrixnli_translate_yaml
+task: afrixnli_translate_yor_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_ibo.yaml

@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: 'Please restore the missing diacritics in the following sentence: {{text}}.
+  Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_ibo_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_wol.yaml

@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: wol
+doc_to_text: 'Please restore the missing diacritics in the following sentence: {{text}}.
+  Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_wol_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_bbj.yaml

@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: bbj
+doc_to_text: "Given a sentence without diacritics, add the appropriate diacritics\
+  \ to make it grammatically and semantically correct. \nSentence: {{text}}. Return\
+  \ output sentence only"
+include: afridiacritics_yaml
+task: afridiacritics_bbj_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_fon.yaml

@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: fon
+doc_to_text: "Given a sentence without diacritics, add the appropriate diacritics\
+  \ to make it grammatically and semantically correct. \nSentence: {{text}}. Return\
+  \ output sentence only"
+include: afridiacritics_yaml
+task: afridiacritics_fon_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_ibo.yaml

@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: 'This text is in Igbo. Restore all diacritical marks to their proper
+  places in the following sentence: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_ibo_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_wol.yaml

@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: wol
+doc_to_text: 'This text is in Wolof. Restore all diacritical marks to their proper
+  places in the following sentence: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_wol_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_yaml

@@ -0,0 +1,25 @@
+tag:
+- adr_tasks
+- adr_prompt_3
+dataset_path: masakhane/diacritics-restoration
+dataset_kwargs: {trust_remote_code: True}
+doc_to_target: target
+output_type: generate_until
+fewshot_split: dev
+test_split: test
+training_split: train
+metric_list:
+  - metric: bleu
+    aggregation: bleu
+    higher_is_better: true
+  - metric: chrf
+    aggregation: chrf
+    higher_is_better: true
+generation_kwargs:
+  do_sample: false
+  until:
+  - '<eos>'
+  - </s>
+  - <|im_end|>
+metadata:
+  version: 1.0

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_yor.yaml

@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: yor
+doc_to_text: 'This text is in Yoruba. Restore all diacritical marks to their proper
+  places in the following sentence: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_yor_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_bbj.yaml

@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: bbj
+doc_to_text: 'You are a linguist specializing in diacritical marks for Gbomala. Add
+  the appropriate diacritics to this Gbomala sentence: {{text}}. Return output sentence
+  only'
+include: afridiacritics_yaml
+task: afridiacritics_bbj_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_fon.yaml

@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: fon
+doc_to_text: 'You are a linguist specializing in diacritical marks for Fon. Add the
+  appropriate diacritics to this Fon sentence: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_fon_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_ibo.yaml

@@ -0,0 +1,6 @@
+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: 'You are a linguist specializing in diacritical marks for Igbo. Add the
+  appropriate diacritics to this Igbo sentence: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_ibo_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_wol.yaml

@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: wol
+doc_to_text: 'You are a linguist specializing in diacritical marks for Wolof. Add
+  the appropriate diacritics to this Wolof sentence: {{text}}. Return output sentence
+  only'
+include: afridiacritics_yaml
+task: afridiacritics_wol_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_yaml

@@ -0,0 +1,25 @@
+tag:
+- adr_tasks
+- adr_prompt_4
+dataset_path: masakhane/diacritics-restoration
+dataset_kwargs: {trust_remote_code: True}
+doc_to_target: target
+output_type: generate_until
+fewshot_split: dev
+test_split: test
+training_split: train
+metric_list:
+  - metric: bleu
+    aggregation: bleu
+    higher_is_better: true
+  - metric: chrf
+    aggregation: chrf
+    higher_is_better: true
+generation_kwargs:
+  do_sample: false
+  until:
+  - '<eos>'
+  - </s>
+  - <|im_end|>
+metadata:
+  version: 1.0

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_ibo.yaml

@@ -0,0 +1,8 @@
+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: 'You are a linguist specializing in diacritical marks for Igbo. Diacritics
+  are essential for proper pronunciation and meaning in Igbo. You are tasked with
+  converting Igbo sentences  without diacritics into their correctly accented forms.
+  Here''s the input: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_ibo_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_wol.yaml

@@ -0,0 +1,8 @@
+# Generated by utils.py
+dataset_name: wol
+doc_to_text: 'You are a linguist specializing in diacritical marks for Wolof. Diacritics
+  are essential for proper pronunciation and meaning in Wolof. You are tasked with
+  converting Wolof sentences  without diacritics into their correctly accented forms.
+  Here''s the input: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_wol_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_yor.yaml

@@ -0,0 +1,8 @@
+# Generated by utils.py
+dataset_name: yor
+doc_to_text: 'You are a linguist specializing in diacritical marks for Yoruba. Diacritics
+  are essential for proper pronunciation and meaning in Yoruba. You are tasked with
+  converting Yoruba sentences  without diacritics into their correctly accented forms.
+  Here''s the input: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_yor_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/README.md

@@ -0,0 +1,24 @@
+#
+
+## Paper
+Title: `AfriQA: Cross-lingual Open-Retrieval Question Answering for African Languages`
+
+Paper Link: https://arxiv.org/abs/2305.06897
+
+## Abstract
+>AfriQA is the first cross-lingual question answering (QA) dataset with a focus on African languages. The dataset includes over 12,000 XOR QA examples across 10 African languages, making it an invaluable resource for developing more equitable QA technology. African languages have historically been underserved in the digital landscape, with far less in-language content available online. This makes it difficult for QA systems to provide accurate information to users in their native language. However, cross-lingual open-retrieval question answering (XOR QA) systems can help fill this gap by retrieving answer content from other languages. AfriQA focuses specifically on African languages where cross-lingual answer content is the only high-coverage source of information. Previous datasets have primarily focused on languages where cross-lingual QA augments coverage from the target language, but AfriQA highlights the importance of African languages as a realistic use case for XOR QA.
+
+HomePage: https://github.com/masakhane-io/afriqa
+
+### Citation
+
+```
+@misc{ogundepo2023afriqa,
+      title={AfriQA: Cross-lingual Open-Retrieval Question Answering for African Languages},
+      author={Odunayo Ogundepo and Tajuddeen R. Gwadabe and Clara E. Rivera and Jonathan H. Clark and Sebastian Ruder and David Ifeoluwa Adelani and Bonaventure F. P. Dossou and Abdou Aziz DIOP and Claytone Sikasote and Gilles Hacheme and Happy Buzaaba and Ignatius Ezeani and Rooweither Mabuya and Salomey Osei and Chris Emezue and Albert Njoroge Kahira and Shamsuddeen H. Muhammad and Akintunde Oladipo and Abraham Toluwase Owodunni and Atnafu Lambebo Tonja and Iyanuoluwa Shode and Akari Asai and Tunde Oluwaseyi Ajayi and Clemencia Siro and Steven Arthur and Mofetoluwa Adeyemi and Orevaoghene Ahia and Aremu Anuoluwapo and Oyinkansola Awosan and Chiamaka Chukwuneke and Bernard Opoku and Awokoya Ayodele and Verrah Otiende and Christine Mwase and Boyd Sinkala and Andre Niyongabo Rubungo and Daniel A. Ajisafe and Emeka Felix Onwuegbuzia and Habib Mbow and Emile Niyomutabazi and Eunice Mukonde and Falalu Ibrahim Lawan and Ibrahim Said Ahmad and Jesujoba O. Alabi and Martin Namukombo and Mbonu Chinedu and Mofya Phiri and Neo Putini and Ndumiso Mngoma and Priscilla A. Amuok and Ruqayya Nasir Iro and Sonia Adhiambo},
+      year={2023},
+      eprint={2305.06897},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/afriqa.yaml

@@ -0,0 +1,13 @@
+group: afriqa
+task:
+  - afriqa_prompt_1
+  - afriqa_prompt_2
+  - afriqa_prompt_3
+  - afriqa_prompt_4
+  - afriqa_prompt_5
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 1

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa

@@ -0,0 +1,42 @@
+tag:
+    - afrobench_xqa_tasks
+    - afriqa_prompt_1
+dataset_kwargs: {trust_remote_code: True}
+dataset_path: masakhane/afriqa-gold-passages
+dataset_name: null
+output_type: generate_until
+test_split: test
+fewshot_split: train
+doc_to_target: answer_pivot
+should_decontaminate: true
+doc_to_decontamination_query: question_lang
+generation_kwargs:
+  until:
+    - "\n"
+  do_sample: false
+  temperature: 0.0
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+target_delimiter: " "
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+      - "."
+      - ","
+      - "\\$"
+  - metric: f1
+    aggregation: !function utils.f1
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+      - "."
+      - ","
+      - "\\$"
+metadata:
+  version: 1.0

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_bem.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: bem
+doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond
+  with the shortest span containing the answer in the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_bem_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_hau.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: hau
+doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond
+  with the shortest span containing the answer in the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_hau_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_ibo.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond
+  with the shortest span containing the answer in the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_ibo_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_kin.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: kin
+doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond
+  with the shortest span containing the answer in the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_kin_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_twi.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: twi
+doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond
+  with the shortest span containing the answer in the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_twi_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_yor.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: yor
+doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond
+  with the shortest span containing the answer in the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_yor_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_zul.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: zul
+doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond
+  with the shortest span containing the answer in the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_zul_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_1/utils.py

@@ -0,0 +1,53 @@
+import re
+import string
+from collections import Counter
+
+
+def normalize_answer(s):
+    """
+    Taken from the official evaluation script for v1.1 of the SQuAD dataset.
+    Lower text and remove punctuation, articles and extra whitespace.
+    """
+
+    def remove_articles(text):
+        return re.sub(r"\b(a|an|the)\b", " ", text)
+
+    def white_space_fix(text):
+        return " ".join(text.split())
+
+    def remove_punc(text):
+        exclude = set(string.punctuation)
+        return "".join(ch for ch in text if ch not in exclude)
+
+    def lower(text):
+        return text.lower()
+
+    return white_space_fix(remove_articles(remove_punc(lower(s))))
+
+
+def f1(items):
+    """
+    Taken from the official evaluation script for v1.1 of the SQuAD dataset.
+    """
+
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+
+    f1_list = []
+
+    for i in range(len(golds)):
+        prediction_tokens = normalize_answer(preds[i]).split()
+        references_tokens = normalize_answer(golds[i]).split()
+        common = Counter(prediction_tokens) & Counter(references_tokens)
+        num_same = sum(common.values())
+        if num_same == 0:
+            f1_score = 0
+        else:
+            precision = 1.0 * num_same / len(prediction_tokens)
+            recall = 1.0 * num_same / len(references_tokens)
+            f1_score = (2 * precision * recall) / (precision + recall)
+
+        f1_list.append(f1_score)
+
+    return sum(f1_list) / len(f1_list)

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa

@@ -0,0 +1,42 @@
+tag:
+    - afrobench_xqa_tasks
+    - afriqa_prompt_2
+dataset_kwargs: {trust_remote_code: True}
+dataset_path: masakhane/afriqa-gold-passages
+dataset_name: null
+output_type: generate_until
+test_split: test
+fewshot_split: train
+doc_to_target: answer_pivot
+should_decontaminate: true
+doc_to_decontamination_query: question_lang
+generation_kwargs:
+  until:
+    - "\n"
+  do_sample: false
+  temperature: 0.0
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+target_delimiter: " "
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+      - "."
+      - ","
+      - "\\$"
+  - metric: f1
+    aggregation: !function utils.f1
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+      - "."
+      - ","
+      - "\\$"
+metadata:
+  version: 1.0

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_bem.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: bem
+doc_to_text: 'Your task is to answer a question given a context. The question is in
+  Bemba, while the context is in English or French.Make sure you respond with the
+  shortest span in the context that contains the answer.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_bem_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_fon.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: fon
+doc_to_text: 'Your task is to answer a question given a context. The question is in
+  Fon, while the context is in English or French.Make sure you respond with the shortest
+  span in the context that contains the answer.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_fon_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_hau.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: hau
+doc_to_text: 'Your task is to answer a question given a context. The question is in
+  Hausa, while the context is in English or French.Make sure you respond with the
+  shortest span in the context that contains the answer.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_hau_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_ibo.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: 'Your task is to answer a question given a context. The question is in
+  Igbo, while the context is in English or French.Make sure you respond with the shortest
+  span in the context that contains the answer.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_ibo_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_kin.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: kin
+doc_to_text: 'Your task is to answer a question given a context. The question is in
+  Kinyarwanda, while the context is in English or French.Make sure you respond with
+  the shortest span in the context that contains the answer.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_kin_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_swa.yaml

@@ -0,0 +1,16 @@
+# Generated by utils.py
+dataset_name: swa
+doc_to_text: 'Your task is to answer a question given a context. The question is in
+  Swahili, while the context is in English or French.Make sure you respond with the
+  shortest span in the context that contains the answer.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+fewshot_split: test
+fewshot_config:
+  sampler: first_n
+task: afriqa_swa_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_twi.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: twi
+doc_to_text: 'Your task is to answer a question given a context. The question is in
+  Twi, while the context is in English or French.Make sure you respond with the shortest
+  span in the context that contains the answer.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_twi_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_yor.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: yor
+doc_to_text: 'Your task is to answer a question given a context. The question is in
+  Yoruba, while the context is in English or French.Make sure you respond with the
+  shortest span in the context that contains the answer.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_yor_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_zul.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: zul
+doc_to_text: 'Your task is to answer a question given a context. The question is in
+  Zulu, while the context is in English or French.Make sure you respond with the shortest
+  span in the context that contains the answer.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_zul_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_2/utils.py

@@ -0,0 +1,53 @@
+import re
+import string
+from collections import Counter
+
+
+def normalize_answer(s):
+    """
+    Taken from the official evaluation script for v1.1 of the SQuAD dataset.
+    Lower text and remove punctuation, articles and extra whitespace.
+    """
+
+    def remove_articles(text):
+        return re.sub(r"\b(a|an|the)\b", " ", text)
+
+    def white_space_fix(text):
+        return " ".join(text.split())
+
+    def remove_punc(text):
+        exclude = set(string.punctuation)
+        return "".join(ch for ch in text if ch not in exclude)
+
+    def lower(text):
+        return text.lower()
+
+    return white_space_fix(remove_articles(remove_punc(lower(s))))
+
+
+def f1(items):
+    """
+    Taken from the official evaluation script for v1.1 of the SQuAD dataset.
+    """
+
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+
+    f1_list = []
+
+    for i in range(len(golds)):
+        prediction_tokens = normalize_answer(preds[i]).split()
+        references_tokens = normalize_answer(golds[i]).split()
+        common = Counter(prediction_tokens) & Counter(references_tokens)
+        num_same = sum(common.values())
+        if num_same == 0:
+            f1_score = 0
+        else:
+            precision = 1.0 * num_same / len(prediction_tokens)
+            recall = 1.0 * num_same / len(references_tokens)
+            f1_score = (2 * precision * recall) / (precision + recall)
+
+        f1_list.append(f1_score)
+
+    return sum(f1_list) / len(f1_list)

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa

@@ -0,0 +1,42 @@
+tag:
+    - afrobench_xqa_tasks
+    - afriqa_prompt_3
+dataset_kwargs: {trust_remote_code: True}
+dataset_path: masakhane/afriqa-gold-passages
+dataset_name: null
+output_type: generate_until
+test_split: test
+fewshot_split: train
+doc_to_target: answer_pivot
+should_decontaminate: true
+doc_to_decontamination_query: question_lang
+generation_kwargs:
+  until:
+    - "\n"
+  do_sample: false
+  temperature: 0.0
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+target_delimiter: " "
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+      - "."
+      - ","
+      - "\\$"
+  - metric: f1
+    aggregation: !function utils.f1
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+      - "."
+      - ","
+      - "\\$"
+metadata:
+  version: 1.0

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_bem.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: bem
+doc_to_text: 'Given the context, provide the answer to the following question.Ensure
+  your response is concise and directly from the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_bem_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_fon.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: fon
+doc_to_text: 'Given the context, provide the answer to the following question.Ensure
+  your response is concise and directly from the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_fon_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_hau.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: hau
+doc_to_text: 'Given the context, provide the answer to the following question.Ensure
+  your response is concise and directly from the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_hau_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_ibo.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: 'Given the context, provide the answer to the following question.Ensure
+  your response is concise and directly from the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_ibo_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_kin.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: kin
+doc_to_text: 'Given the context, provide the answer to the following question.Ensure
+  your response is concise and directly from the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_kin_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_swa.yaml

@@ -0,0 +1,15 @@
+# Generated by utils.py
+dataset_name: swa
+doc_to_text: 'Given the context, provide the answer to the following question.Ensure
+  your response is concise and directly from the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+fewshot_split: test
+fewshot_config:
+  sampler: first_n
+task: afriqa_swa_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_twi.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: twi
+doc_to_text: 'Given the context, provide the answer to the following question.Ensure
+  your response is concise and directly from the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_twi_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_yor.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: yor
+doc_to_text: 'Given the context, provide the answer to the following question.Ensure
+  your response is concise and directly from the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_yor_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_zul.yaml

@@ -0,0 +1,12 @@
+# Generated by utils.py
+dataset_name: zul
+doc_to_text: 'Given the context, provide the answer to the following question.Ensure
+  your response is concise and directly from the context.
+
+  Question: {{question_lang}}
+
+  Context: {{context}}
+
+  Answer:'
+include: afriqa
+task: afriqa_zul_prompt_3