Add files using upload-large-folder tool

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_fon.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: fon
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_fon_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/utils.py

@@ -0,0 +1,55 @@
+from itertools import chain
+
+from sklearn.metrics import accuracy_score
+
+from lm_eval.utils import weighted_f1_score
+
+
+def doc_to_target(doc):
+    pos_tag_map = {
+        0: "NOUN",
+        1: "PUNCT",
+        2: "ADP",
+        3: "NUM",
+        4: "SYM",
+        5: "SCONJ",
+        6: "ADJ",
+        7: "PART",
+        8: "DET",
+        9: "CCONJ",
+        10: "PROPN",
+        11: "PRON",
+        12: "X",
+        13: "_",
+        14: "ADV",
+        15: "INTJ",
+        16: "VERB",
+        17: "AUX",
+    }
+    return [pos_tag_map[tag] for tag in doc["upos"]]
+
+
+def acc_score(items):
+    unzipped_list = list(zip(*items))
+
+    golds, preds = unzipped_list[0], unzipped_list[1]
+
+    # Flatten preds' inner lists
+    flattened_preds = [list(chain.from_iterable(p)) for p in preds]
+
+    # Calculate the accuracy for each gold-pred pair
+    accuracy_scores = []
+    for gold, pred in zip(golds, flattened_preds):
+        # Ensure both lists are of the same length, otherwise truncate to match
+        min_length = min(len(gold), len(pred))
+        gold = gold[:min_length]
+        pred = pred[:min_length]
+
+        # Calculate accuracy for the current pair and add to the list
+        accuracy = accuracy_score(gold, pred)
+        accuracy_scores.append(accuracy)
+
+    mean_accuracy = (
+        sum(accuracy_scores) / len(accuracy_scores) if accuracy_scores else 0
+    )
+    return mean_accuracy

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_ibo.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: "You are an expert in tagging words and sentences in Igbo with the right\
+  \ POS tag. \n\nPlease provide the POS tags for each word in the Igbo sentence. The\
+  \ input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP', 'ADV',\
+  \ 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\
+  \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\
+  \ each tuple consists of a word from the input text and its corresponding POS tag\
+  \ label from the POS tag label set provided\nYour response should include only a\
+  \ list of tuples, in the order that the words appear in the input sentence, including\
+  \ punctuations, with each tuple containing the corresponding POS tag label for a\
+  \ word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_ibo_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_kin.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: kin
+doc_to_text: "You are an expert in tagging words and sentences in Kinyarwanda with\
+  \ the right POS tag. \n\nPlease provide the POS tags for each word in the Kinyarwanda\
+  \ sentence. The input is a list of words in the sentence. POS tag label set: ['ADJ',\
+  \ 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN',\
+  \ 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples,\
+  \ where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the POS tag label set provided\nYour response should include\
+  \ only a list of tuples, in the order that the words appear in the input sentence,\
+  \ including punctuations, with each tuple containing the corresponding POS tag label\
+  \ for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_kin_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_lug.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: lug
+doc_to_text: "You are an expert in tagging words and sentences in Luganda with the\
+  \ right POS tag. \n\nPlease provide the POS tags for each word in the Luganda sentence.\
+  \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\
+  \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\
+  \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\
+  \ each tuple consists of a word from the input text and its corresponding POS tag\
+  \ label from the POS tag label set provided\nYour response should include only a\
+  \ list of tuples, in the order that the words appear in the input sentence, including\
+  \ punctuations, with each tuple containing the corresponding POS tag label for a\
+  \ word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_lug_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_sna.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: sna
+doc_to_text: "You are an expert in tagging words and sentences in chiShona with the\
+  \ right POS tag. \n\nPlease provide the POS tags for each word in the chiShona sentence.\
+  \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\
+  \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\
+  \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\
+  \ each tuple consists of a word from the input text and its corresponding POS tag\
+  \ label from the POS tag label set provided\nYour response should include only a\
+  \ list of tuples, in the order that the words appear in the input sentence, including\
+  \ punctuations, with each tuple containing the corresponding POS tag label for a\
+  \ word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_sna_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_tsn.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: tsn
+doc_to_text: "You are an expert in tagging words and sentences in Setswana with the\
+  \ right POS tag. \n\nPlease provide the POS tags for each word in the Setswana sentence.\
+  \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\
+  \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\
+  \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\
+  \ each tuple consists of a word from the input text and its corresponding POS tag\
+  \ label from the POS tag label set provided\nYour response should include only a\
+  \ list of tuples, in the order that the words appear in the input sentence, including\
+  \ punctuations, with each tuple containing the corresponding POS tag label for a\
+  \ word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_tsn_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_wol.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: wol
+doc_to_text: "You are an expert in tagging words and sentences in Wolof with the right\
+  \ POS tag. \n\nPlease provide the POS tags for each word in the Wolof sentence.\
+  \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\
+  \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\
+  \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\
+  \ each tuple consists of a word from the input text and its corresponding POS tag\
+  \ label from the POS tag label set provided\nYour response should include only a\
+  \ list of tuples, in the order that the words appear in the input sentence, including\
+  \ punctuations, with each tuple containing the corresponding POS tag label for a\
+  \ word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_wol_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_yaml

@@ -0,0 +1,32 @@
+tag:
+- masakhapos_tasks
+- masakhapos_prompt_2
+dataset_path: masakhane/masakhapos
+dataset_name: null
+dataset_kwargs: {trust_remote_code: True}
+output_type: generate_until
+generation_kwargs:
+  do_sample: false
+  until:
+  - </s>
+  - <|im_end|>
+validation_split: validation
+test_split: test
+fewshot_split: train
+doc_to_target: !function utils.doc_to_target
+should_decontaminate: true
+doc_to_decontamination_query: "Sentence: {{token}}\nOutput:"
+filter_list:
+  - filter:
+    - function: regex_pos
+    name: flexible-extract
+metric_list:
+  - metric: acc
+    aggregation: !function utils.acc_score
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+      - ","
+metadata:
+  version: 1.0

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_zul.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: zul
+doc_to_text: "You are an expert in tagging words and sentences in isiZulu with the\
+  \ right POS tag. \n\nPlease provide the POS tags for each word in the isiZulu sentence.\
+  \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\
+  \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\
+  \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\
+  \ each tuple consists of a word from the input text and its corresponding POS tag\
+  \ label from the POS tag label set provided\nYour response should include only a\
+  \ list of tuples, in the order that the words appear in the input sentence, including\
+  \ punctuations, with each tuple containing the corresponding POS tag label for a\
+  \ word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_zul_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_2/utils.py

@@ -0,0 +1,55 @@
+from itertools import chain
+
+from sklearn.metrics import accuracy_score
+
+from lm_eval.utils import weighted_f1_score
+
+
+def doc_to_target(doc):
+    pos_tag_map = {
+        0: "NOUN",
+        1: "PUNCT",
+        2: "ADP",
+        3: "NUM",
+        4: "SYM",
+        5: "SCONJ",
+        6: "ADJ",
+        7: "PART",
+        8: "DET",
+        9: "CCONJ",
+        10: "PROPN",
+        11: "PRON",
+        12: "X",
+        13: "_",
+        14: "ADV",
+        15: "INTJ",
+        16: "VERB",
+        17: "AUX",
+    }
+    return [pos_tag_map[tag] for tag in doc["upos"]]
+
+
+def acc_score(items):
+    unzipped_list = list(zip(*items))
+
+    golds, preds = unzipped_list[0], unzipped_list[1]
+
+    # Flatten preds' inner lists
+    flattened_preds = [list(chain.from_iterable(p)) for p in preds]
+
+    # Calculate the accuracy for each gold-pred pair
+    accuracy_scores = []
+    for gold, pred in zip(golds, flattened_preds):
+        # Ensure both lists are of the same length, otherwise truncate to match
+        min_length = min(len(gold), len(pred))
+        gold = gold[:min_length]
+        pred = pred[:min_length]
+
+        # Calculate accuracy for the current pair and add to the list
+        accuracy = accuracy_score(gold, pred)
+        accuracy_scores.append(accuracy)
+
+    mean_accuracy = (
+        sum(accuracy_scores) / len(accuracy_scores) if accuracy_scores else 0
+    )
+    return mean_accuracy

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_bbj.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: bbj
+doc_to_text: "Acting as a Ghomala linguist and without making any corrections or changes\
+  \ to the text, perform a part of speech (POS) analysis of the sentences using the\
+  \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input will be a list of words in the sentence. The output format should\
+  \ be a list of tuples, where each tuple consists of a word from the input text and\
+  \ its corresponding POS tag label from the POS tag label set provided\nYour response\
+  \ should include only a list of tuples, in the order that the words appear in the\
+  \ input sentence, including punctuations, with each tuple containing the corresponding\
+  \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_bbj_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_ewe.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: ewe
+doc_to_text: "Acting as a Ewe linguist and without making any corrections or changes\
+  \ to the text, perform a part of speech (POS) analysis of the sentences using the\
+  \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input will be a list of words in the sentence. The output format should\
+  \ be a list of tuples, where each tuple consists of a word from the input text and\
+  \ its corresponding POS tag label from the POS tag label set provided\nYour response\
+  \ should include only a list of tuples, in the order that the words appear in the\
+  \ input sentence, including punctuations, with each tuple containing the corresponding\
+  \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_ewe_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_fon.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: fon
+doc_to_text: "Acting as a Fon linguist and without making any corrections or changes\
+  \ to the text, perform a part of speech (POS) analysis of the sentences using the\
+  \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input will be a list of words in the sentence. The output format should\
+  \ be a list of tuples, where each tuple consists of a word from the input text and\
+  \ its corresponding POS tag label from the POS tag label set provided\nYour response\
+  \ should include only a list of tuples, in the order that the words appear in the\
+  \ input sentence, including punctuations, with each tuple containing the corresponding\
+  \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_fon_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_ibo.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: "Acting as a Igbo linguist and without making any corrections or changes\
+  \ to the text, perform a part of speech (POS) analysis of the sentences using the\
+  \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input will be a list of words in the sentence. The output format should\
+  \ be a list of tuples, where each tuple consists of a word from the input text and\
+  \ its corresponding POS tag label from the POS tag label set provided\nYour response\
+  \ should include only a list of tuples, in the order that the words appear in the\
+  \ input sentence, including punctuations, with each tuple containing the corresponding\
+  \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_ibo_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_kin.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: kin
+doc_to_text: "Acting as a Kinyarwanda linguist and without making any corrections\
+  \ or changes to the text, perform a part of speech (POS) analysis of the sentences\
+  \ using the following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ,\
+  \ 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',\
+  \ 'VERB', 'X']. The input will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_kin_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_lug.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: lug
+doc_to_text: "Acting as a Luganda linguist and without making any corrections or changes\
+  \ to the text, perform a part of speech (POS) analysis of the sentences using the\
+  \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input will be a list of words in the sentence. The output format should\
+  \ be a list of tuples, where each tuple consists of a word from the input text and\
+  \ its corresponding POS tag label from the POS tag label set provided\nYour response\
+  \ should include only a list of tuples, in the order that the words appear in the\
+  \ input sentence, including punctuations, with each tuple containing the corresponding\
+  \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_lug_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_luo.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: luo
+doc_to_text: "Acting as a Dholuo linguist and without making any corrections or changes\
+  \ to the text, perform a part of speech (POS) analysis of the sentences using the\
+  \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input will be a list of words in the sentence. The output format should\
+  \ be a list of tuples, where each tuple consists of a word from the input text and\
+  \ its corresponding POS tag label from the POS tag label set provided\nYour response\
+  \ should include only a list of tuples, in the order that the words appear in the\
+  \ input sentence, including punctuations, with each tuple containing the corresponding\
+  \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_luo_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_mos.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: mos
+doc_to_text: "Acting as a Mossi linguist and without making any corrections or changes\
+  \ to the text, perform a part of speech (POS) analysis of the sentences using the\
+  \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input will be a list of words in the sentence. The output format should\
+  \ be a list of tuples, where each tuple consists of a word from the input text and\
+  \ its corresponding POS tag label from the POS tag label set provided\nYour response\
+  \ should include only a list of tuples, in the order that the words appear in the\
+  \ input sentence, including punctuations, with each tuple containing the corresponding\
+  \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_mos_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_pcm.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: pcm
+doc_to_text: "Acting as a Nigerian Pidgin linguist and without making any corrections\
+  \ or changes to the text, perform a part of speech (POS) analysis of the sentences\
+  \ using the following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ,\
+  \ 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',\
+  \ 'VERB', 'X']. The input will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_pcm_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_sna.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: sna
+doc_to_text: "Acting as a chiShona linguist and without making any corrections or\
+  \ changes to the text, perform a part of speech (POS) analysis of the sentences\
+  \ using the following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ,\
+  \ 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',\
+  \ 'VERB', 'X']. The input will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_sna_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_swa.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: swa
+doc_to_text: "Acting as a Kiswahili linguist and without making any corrections or\
+  \ changes to the text, perform a part of speech (POS) analysis of the sentences\
+  \ using the following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ,\
+  \ 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',\
+  \ 'VERB', 'X']. The input will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_swa_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_twi.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: twi
+doc_to_text: "Acting as a Twi linguist and without making any corrections or changes\
+  \ to the text, perform a part of speech (POS) analysis of the sentences using the\
+  \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input will be a list of words in the sentence. The output format should\
+  \ be a list of tuples, where each tuple consists of a word from the input text and\
+  \ its corresponding POS tag label from the POS tag label set provided\nYour response\
+  \ should include only a list of tuples, in the order that the words appear in the\
+  \ input sentence, including punctuations, with each tuple containing the corresponding\
+  \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_twi_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_wol.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: wol
+doc_to_text: "Acting as a Wolof linguist and without making any corrections or changes\
+  \ to the text, perform a part of speech (POS) analysis of the sentences using the\
+  \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input will be a list of words in the sentence. The output format should\
+  \ be a list of tuples, where each tuple consists of a word from the input text and\
+  \ its corresponding POS tag label from the POS tag label set provided\nYour response\
+  \ should include only a list of tuples, in the order that the words appear in the\
+  \ input sentence, including punctuations, with each tuple containing the corresponding\
+  \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_wol_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_xho.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: xho
+doc_to_text: "Acting as a isiXhosa linguist and without making any corrections or\
+  \ changes to the text, perform a part of speech (POS) analysis of the sentences\
+  \ using the following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ,\
+  \ 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',\
+  \ 'VERB', 'X']. The input will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_xho_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_yaml

@@ -0,0 +1,32 @@
+tag:
+- masakhapos_tasks
+- masakhapos_prompt_3
+dataset_path: masakhane/masakhapos
+dataset_name: null
+dataset_kwargs: {trust_remote_code: True}
+output_type: generate_until
+generation_kwargs:
+  do_sample: false
+  until:
+  - </s>
+  - <|im_end|>
+validation_split: validation
+test_split: test
+fewshot_split: train
+doc_to_target: !function utils.doc_to_target
+should_decontaminate: true
+doc_to_decontamination_query: "Sentence: {{token}}\nOutput:"
+filter_list:
+  - filter:
+    - function: regex_pos
+    name: flexible-extract
+metric_list:
+  - metric: acc
+    aggregation: !function utils.acc_score
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+      - ","
+metadata:
+  version: 1.0

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_yor.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: yor
+doc_to_text: "Acting as a Yoruba linguist and without making any corrections or changes\
+  \ to the text, perform a part of speech (POS) analysis of the sentences using the\
+  \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input will be a list of words in the sentence. The output format should\
+  \ be a list of tuples, where each tuple consists of a word from the input text and\
+  \ its corresponding POS tag label from the POS tag label set provided\nYour response\
+  \ should include only a list of tuples, in the order that the words appear in the\
+  \ input sentence, including punctuations, with each tuple containing the corresponding\
+  \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_yor_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_zul.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: zul
+doc_to_text: "Acting as a isiZulu linguist and without making any corrections or changes\
+  \ to the text, perform a part of speech (POS) analysis of the sentences using the\
+  \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input will be a list of words in the sentence. The output format should\
+  \ be a list of tuples, where each tuple consists of a word from the input text and\
+  \ its corresponding POS tag label from the POS tag label set provided\nYour response\
+  \ should include only a list of tuples, in the order that the words appear in the\
+  \ input sentence, including punctuations, with each tuple containing the corresponding\
+  \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_zul_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_3/utils.py

@@ -0,0 +1,55 @@
+from itertools import chain
+
+from sklearn.metrics import accuracy_score
+
+from lm_eval.utils import weighted_f1_score
+
+
+def doc_to_target(doc):
+    pos_tag_map = {
+        0: "NOUN",
+        1: "PUNCT",
+        2: "ADP",
+        3: "NUM",
+        4: "SYM",
+        5: "SCONJ",
+        6: "ADJ",
+        7: "PART",
+        8: "DET",
+        9: "CCONJ",
+        10: "PROPN",
+        11: "PRON",
+        12: "X",
+        13: "_",
+        14: "ADV",
+        15: "INTJ",
+        16: "VERB",
+        17: "AUX",
+    }
+    return [pos_tag_map[tag] for tag in doc["upos"]]
+
+
+def acc_score(items):
+    unzipped_list = list(zip(*items))
+
+    golds, preds = unzipped_list[0], unzipped_list[1]
+
+    # Flatten preds' inner lists
+    flattened_preds = [list(chain.from_iterable(p)) for p in preds]
+
+    # Calculate the accuracy for each gold-pred pair
+    accuracy_scores = []
+    for gold, pred in zip(golds, flattened_preds):
+        # Ensure both lists are of the same length, otherwise truncate to match
+        min_length = min(len(gold), len(pred))
+        gold = gold[:min_length]
+        pred = pred[:min_length]
+
+        # Calculate accuracy for the current pair and add to the list
+        accuracy = accuracy_score(gold, pred)
+        accuracy_scores.append(accuracy)
+
+    mean_accuracy = (
+        sum(accuracy_scores) / len(accuracy_scores) if accuracy_scores else 0
+    )
+    return mean_accuracy

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_bam.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: bam
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_bam_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_ewe.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: ewe
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_ewe_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_fon.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: fon
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_fon_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_hau.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: hau
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_hau_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_ibo.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_ibo_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_kin.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: kin
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_kin_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_lug.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: lug
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_lug_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_luo.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: luo
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_luo_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_mos.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: mos
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_mos_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_nya.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: nya
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_nya_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_pcm.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: pcm
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_pcm_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_sna.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: sna
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_sna_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_swa.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: swa
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_swa_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_tsn.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: tsn
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_tsn_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_twi.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: twi
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_twi_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_wol.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: wol
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_wol_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_xho.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: xho
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_xho_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_yaml

@@ -0,0 +1,32 @@
+tag:
+- masakhapos_tasks
+- masakhapos_prompt_4
+dataset_path: masakhane/masakhapos
+dataset_name: null
+dataset_kwargs: {trust_remote_code: True}
+output_type: generate_until
+generation_kwargs:
+  do_sample: false
+  until:
+  - </s>
+  - <|im_end|>
+validation_split: validation
+test_split: test
+fewshot_split: train
+doc_to_target: !function utils.doc_to_target
+should_decontaminate: true
+doc_to_decontamination_query: "Sentence: {{token}}\nOutput:"
+filter_list:
+  - filter:
+    - function: regex_pos
+    name: flexible-extract
+metric_list:
+  - metric: acc
+    aggregation: !function utils.acc_score
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+      - ","
+metadata:
+  version: 1.0

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_yor.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: yor
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_yor_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_zul.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: zul
+doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\
+  \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. The input sentence will be a list of words in the sentence. The output format\
+  \ should be a list of tuples, where each tuple consists of a word from the input\
+  \ text and its corresponding POS tag label from the POS tag label set provided\n\
+  Your response should include only a list of tuples, in the order that the words\
+  \ appear in the input sentence, including punctuations, with each tuple containing\
+  \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_zul_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_4/utils.py

@@ -0,0 +1,55 @@
+from itertools import chain
+
+from sklearn.metrics import accuracy_score
+
+from lm_eval.utils import weighted_f1_score
+
+
+def doc_to_target(doc):
+    pos_tag_map = {
+        0: "NOUN",
+        1: "PUNCT",
+        2: "ADP",
+        3: "NUM",
+        4: "SYM",
+        5: "SCONJ",
+        6: "ADJ",
+        7: "PART",
+        8: "DET",
+        9: "CCONJ",
+        10: "PROPN",
+        11: "PRON",
+        12: "X",
+        13: "_",
+        14: "ADV",
+        15: "INTJ",
+        16: "VERB",
+        17: "AUX",
+    }
+    return [pos_tag_map[tag] for tag in doc["upos"]]
+
+
+def acc_score(items):
+    unzipped_list = list(zip(*items))
+
+    golds, preds = unzipped_list[0], unzipped_list[1]
+
+    # Flatten preds' inner lists
+    flattened_preds = [list(chain.from_iterable(p)) for p in preds]
+
+    # Calculate the accuracy for each gold-pred pair
+    accuracy_scores = []
+    for gold, pred in zip(golds, flattened_preds):
+        # Ensure both lists are of the same length, otherwise truncate to match
+        min_length = min(len(gold), len(pred))
+        gold = gold[:min_length]
+        pred = pred[:min_length]
+
+        # Calculate accuracy for the current pair and add to the list
+        accuracy = accuracy_score(gold, pred)
+        accuracy_scores.append(accuracy)
+
+    mean_accuracy = (
+        sum(accuracy_scores) / len(accuracy_scores) if accuracy_scores else 0
+    )
+    return mean_accuracy