Add files using upload-large-folder tool

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_tir.yaml

@@ -0,0 +1,8 @@
+# Generated by utils.py
+dataset_name: tir
+doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\
+  \ categories technology, religion, politics, sports, health, entertainment, or business;\
+  \ what is the topic of the Tigrinya statement below? Return only the category. \n\
+  \ntext: {{headline_text}} \\category:\n\n"
+include: masakhanews
+task: masakhanews_tir_prompt_3

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_3/utils.py

@@ -0,0 +1 @@
+from lm_eval.utils import weighted_f1_score

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_orm.yaml

@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: orm
+doc_to_text: "Label the following text as technology, religion, politics, sports,\
+  \ health, entertainment, or geography. Provide only the category as your response.\
+  \ \n\ntext: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_orm_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_sna.yaml

@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: sna
+doc_to_text: "Label the following text as technology, religion, politics, sports,\
+  \ health, entertainment, or geography. Provide only the category as your response.\
+  \ \n\ntext: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_sna_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_som.yaml

@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: som
+doc_to_text: "Label the following text as technology, religion, politics, sports,\
+  \ health, entertainment, or geography. Provide only the category as your response.\
+  \ \n\ntext: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_som_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_swa.yaml

@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: swa
+doc_to_text: "Label the following text as technology, religion, politics, sports,\
+  \ health, entertainment, or geography. Provide only the category as your response.\
+  \ \n\ntext: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_swa_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_tir.yaml

@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: tir
+doc_to_text: "Label the following text as technology, religion, politics, sports,\
+  \ health, entertainment, or geography. Provide only the category as your response.\
+  \ \n\ntext: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_tir_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_xho.yaml

@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: xho
+doc_to_text: "Label the following text as technology, religion, politics, sports,\
+  \ health, entertainment, or geography. Provide only the category as your response.\
+  \ \n\ntext: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_xho_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_yor.yaml

@@ -0,0 +1,7 @@
+# Generated by utils.py
+dataset_name: yor
+doc_to_text: "Label the following text as technology, religion, politics, sports,\
+  \ health, entertainment, or geography. Provide only the category as your response.\
+  \ \n\ntext: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_yor_prompt_4

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_4/utils.py

@@ -0,0 +1 @@
+from lm_eval.utils import weighted_f1_score

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_amh.yaml

@@ -0,0 +1,18 @@
+# Generated by utils.py
+dataset_name: amh
+doc_to_text: "You are tasked with performing topic classification on the following\
+  \ Amharic text. For each input, classify the topic as technology, business, politics,\
+  \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\
+  \ technology: The text discusses scientific discoveries, technological advancements,\
+  \ or related topics. \npolitics: The text covers political events, policies, or\
+  \ related topics. \nsports: The text talks about sports events, athletes, or related\
+  \ topics. \nhealth: The text addresses health issues, medical advancements, or related\
+  \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\
+  \ topics. \nreligion: The text talks about relgions, religious institutions and\
+  \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\
+  \ related topics. \n\nIf the text contains multiple topics, choose the dominant\
+  \ topic. For ambiguous or unclear topics, select the category that best reflects\
+  \ the overall content. Please provide a single classification for each input.\n\n\
+  text: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_amh_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_eng.yaml

@@ -0,0 +1,18 @@
+# Generated by utils.py
+dataset_name: eng
+doc_to_text: "You are tasked with performing topic classification on the following\
+  \ English text. For each input, classify the topic as technology, business, politics,\
+  \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\
+  \ technology: The text discusses scientific discoveries, technological advancements,\
+  \ or related topics. \npolitics: The text covers political events, policies, or\
+  \ related topics. \nsports: The text talks about sports events, athletes, or related\
+  \ topics. \nhealth: The text addresses health issues, medical advancements, or related\
+  \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\
+  \ topics. \nreligion: The text talks about relgions, religious institutions and\
+  \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\
+  \ related topics. \n\nIf the text contains multiple topics, choose the dominant\
+  \ topic. For ambiguous or unclear topics, select the category that best reflects\
+  \ the overall content. Please provide a single classification for each input.\n\n\
+  text: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_eng_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_ibo.yaml

@@ -0,0 +1,18 @@
+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: "You are tasked with performing topic classification on the following\
+  \ Igbo text. For each input, classify the topic as technology, business, politics,\
+  \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\
+  \ technology: The text discusses scientific discoveries, technological advancements,\
+  \ or related topics. \npolitics: The text covers political events, policies, or\
+  \ related topics. \nsports: The text talks about sports events, athletes, or related\
+  \ topics. \nhealth: The text addresses health issues, medical advancements, or related\
+  \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\
+  \ topics. \nreligion: The text talks about relgions, religious institutions and\
+  \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\
+  \ related topics. \n\nIf the text contains multiple topics, choose the dominant\
+  \ topic. For ambiguous or unclear topics, select the category that best reflects\
+  \ the overall content. Please provide a single classification for each input.\n\n\
+  text: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_ibo_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_lug.yaml

@@ -0,0 +1,18 @@
+# Generated by utils.py
+dataset_name: lug
+doc_to_text: "You are tasked with performing topic classification on the following\
+  \ Luganda text. For each input, classify the topic as technology, business, politics,\
+  \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\
+  \ technology: The text discusses scientific discoveries, technological advancements,\
+  \ or related topics. \npolitics: The text covers political events, policies, or\
+  \ related topics. \nsports: The text talks about sports events, athletes, or related\
+  \ topics. \nhealth: The text addresses health issues, medical advancements, or related\
+  \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\
+  \ topics. \nreligion: The text talks about relgions, religious institutions and\
+  \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\
+  \ related topics. \n\nIf the text contains multiple topics, choose the dominant\
+  \ topic. For ambiguous or unclear topics, select the category that best reflects\
+  \ the overall content. Please provide a single classification for each input.\n\n\
+  text: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_lug_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_orm.yaml

@@ -0,0 +1,18 @@
+# Generated by utils.py
+dataset_name: orm
+doc_to_text: "You are tasked with performing topic classification on the following\
+  \ Afaan Oromoo text. For each input, classify the topic as technology, business,\
+  \ politics, sports, health, entertainment, or religion. Use the following guidelines:\
+  \ \n\n technology: The text discusses scientific discoveries, technological advancements,\
+  \ or related topics. \npolitics: The text covers political events, policies, or\
+  \ related topics. \nsports: The text talks about sports events, athletes, or related\
+  \ topics. \nhealth: The text addresses health issues, medical advancements, or related\
+  \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\
+  \ topics. \nreligion: The text talks about relgions, religious institutions and\
+  \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\
+  \ related topics. \n\nIf the text contains multiple topics, choose the dominant\
+  \ topic. For ambiguous or unclear topics, select the category that best reflects\
+  \ the overall content. Please provide a single classification for each input.\n\n\
+  text: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_orm_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_pcm.yaml

@@ -0,0 +1,18 @@
+# Generated by utils.py
+dataset_name: pcm
+doc_to_text: "You are tasked with performing topic classification on the following\
+  \ Nigerian Pidgin text. For each input, classify the topic as technology, business,\
+  \ politics, sports, health, entertainment, or religion. Use the following guidelines:\
+  \ \n\n technology: The text discusses scientific discoveries, technological advancements,\
+  \ or related topics. \npolitics: The text covers political events, policies, or\
+  \ related topics. \nsports: The text talks about sports events, athletes, or related\
+  \ topics. \nhealth: The text addresses health issues, medical advancements, or related\
+  \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\
+  \ topics. \nreligion: The text talks about relgions, religious institutions and\
+  \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\
+  \ related topics. \n\nIf the text contains multiple topics, choose the dominant\
+  \ topic. For ambiguous or unclear topics, select the category that best reflects\
+  \ the overall content. Please provide a single classification for each input.\n\n\
+  text: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_pcm_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_run.yaml

@@ -0,0 +1,18 @@
+# Generated by utils.py
+dataset_name: run
+doc_to_text: "You are tasked with performing topic classification on the following\
+  \ Kirundi text. For each input, classify the topic as technology, business, politics,\
+  \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\
+  \ technology: The text discusses scientific discoveries, technological advancements,\
+  \ or related topics. \npolitics: The text covers political events, policies, or\
+  \ related topics. \nsports: The text talks about sports events, athletes, or related\
+  \ topics. \nhealth: The text addresses health issues, medical advancements, or related\
+  \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\
+  \ topics. \nreligion: The text talks about relgions, religious institutions and\
+  \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\
+  \ related topics. \n\nIf the text contains multiple topics, choose the dominant\
+  \ topic. For ambiguous or unclear topics, select the category that best reflects\
+  \ the overall content. Please provide a single classification for each input.\n\n\
+  text: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_run_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_sna.yaml

@@ -0,0 +1,18 @@
+# Generated by utils.py
+dataset_name: sna
+doc_to_text: "You are tasked with performing topic classification on the following\
+  \ Shona text. For each input, classify the topic as technology, business, politics,\
+  \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\
+  \ technology: The text discusses scientific discoveries, technological advancements,\
+  \ or related topics. \npolitics: The text covers political events, policies, or\
+  \ related topics. \nsports: The text talks about sports events, athletes, or related\
+  \ topics. \nhealth: The text addresses health issues, medical advancements, or related\
+  \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\
+  \ topics. \nreligion: The text talks about relgions, religious institutions and\
+  \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\
+  \ related topics. \n\nIf the text contains multiple topics, choose the dominant\
+  \ topic. For ambiguous or unclear topics, select the category that best reflects\
+  \ the overall content. Please provide a single classification for each input.\n\n\
+  text: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_sna_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_som.yaml

@@ -0,0 +1,18 @@
+# Generated by utils.py
+dataset_name: som
+doc_to_text: "You are tasked with performing topic classification on the following\
+  \ Somali text. For each input, classify the topic as technology, business, politics,\
+  \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\
+  \ technology: The text discusses scientific discoveries, technological advancements,\
+  \ or related topics. \npolitics: The text covers political events, policies, or\
+  \ related topics. \nsports: The text talks about sports events, athletes, or related\
+  \ topics. \nhealth: The text addresses health issues, medical advancements, or related\
+  \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\
+  \ topics. \nreligion: The text talks about relgions, religious institutions and\
+  \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\
+  \ related topics. \n\nIf the text contains multiple topics, choose the dominant\
+  \ topic. For ambiguous or unclear topics, select the category that best reflects\
+  \ the overall content. Please provide a single classification for each input.\n\n\
+  text: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_som_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_swa.yaml

@@ -0,0 +1,18 @@
+# Generated by utils.py
+dataset_name: swa
+doc_to_text: "You are tasked with performing topic classification on the following\
+  \ Swahili text. For each input, classify the topic as technology, business, politics,\
+  \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\
+  \ technology: The text discusses scientific discoveries, technological advancements,\
+  \ or related topics. \npolitics: The text covers political events, policies, or\
+  \ related topics. \nsports: The text talks about sports events, athletes, or related\
+  \ topics. \nhealth: The text addresses health issues, medical advancements, or related\
+  \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\
+  \ topics. \nreligion: The text talks about relgions, religious institutions and\
+  \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\
+  \ related topics. \n\nIf the text contains multiple topics, choose the dominant\
+  \ topic. For ambiguous or unclear topics, select the category that best reflects\
+  \ the overall content. Please provide a single classification for each input.\n\n\
+  text: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_swa_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_tir.yaml

@@ -0,0 +1,18 @@
+# Generated by utils.py
+dataset_name: tir
+doc_to_text: "You are tasked with performing topic classification on the following\
+  \ Tigrinya text. For each input, classify the topic as technology, business, politics,\
+  \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\
+  \ technology: The text discusses scientific discoveries, technological advancements,\
+  \ or related topics. \npolitics: The text covers political events, policies, or\
+  \ related topics. \nsports: The text talks about sports events, athletes, or related\
+  \ topics. \nhealth: The text addresses health issues, medical advancements, or related\
+  \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\
+  \ topics. \nreligion: The text talks about relgions, religious institutions and\
+  \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\
+  \ related topics. \n\nIf the text contains multiple topics, choose the dominant\
+  \ topic. For ambiguous or unclear topics, select the category that best reflects\
+  \ the overall content. Please provide a single classification for each input.\n\n\
+  text: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_tir_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_xho.yaml

@@ -0,0 +1,18 @@
+# Generated by utils.py
+dataset_name: xho
+doc_to_text: "You are tasked with performing topic classification on the following\
+  \ Xhosa text. For each input, classify the topic as technology, business, politics,\
+  \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\
+  \ technology: The text discusses scientific discoveries, technological advancements,\
+  \ or related topics. \npolitics: The text covers political events, policies, or\
+  \ related topics. \nsports: The text talks about sports events, athletes, or related\
+  \ topics. \nhealth: The text addresses health issues, medical advancements, or related\
+  \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\
+  \ topics. \nreligion: The text talks about relgions, religious institutions and\
+  \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\
+  \ related topics. \n\nIf the text contains multiple topics, choose the dominant\
+  \ topic. For ambiguous or unclear topics, select the category that best reflects\
+  \ the overall content. Please provide a single classification for each input.\n\n\
+  text: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_xho_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_yor.yaml

@@ -0,0 +1,18 @@
+# Generated by utils.py
+dataset_name: yor
+doc_to_text: "You are tasked with performing topic classification on the following\
+  \ Yoruba text. For each input, classify the topic as technology, business, politics,\
+  \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\
+  \ technology: The text discusses scientific discoveries, technological advancements,\
+  \ or related topics. \npolitics: The text covers political events, policies, or\
+  \ related topics. \nsports: The text talks about sports events, athletes, or related\
+  \ topics. \nhealth: The text addresses health issues, medical advancements, or related\
+  \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\
+  \ topics. \nreligion: The text talks about relgions, religious institutions and\
+  \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\
+  \ related topics. \n\nIf the text contains multiple topics, choose the dominant\
+  \ topic. For ambiguous or unclear topics, select the category that best reflects\
+  \ the overall content. Please provide a single classification for each input.\n\n\
+  text: {{headline_text}} \\category: \n\n"
+include: masakhanews
+task: masakhanews_yor_prompt_5

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhanews/prompt_5/utils.py

@@ -0,0 +1 @@
+from lm_eval.utils import weighted_f1_score

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/README.md

@@ -0,0 +1,75 @@
+#
+
+## Paper
+Title: `MasakhaPOS: Part-of-Speech Tagging for Typologically Diverse African languages`
+
+Paper Link: https://aclanthology.org/2023.acl-long.609/
+
+## Abstract
+>In this paper, we present AfricaPOS, the largest part-of-speech (POS) dataset for 20 typologically diverse African languages. We discuss the challenges in annotating POS for these languages using the universal dependencies (UD) guidelines. We conducted extensive POS baseline experiments using both conditional random field and several multilingual pre-trained language models. We applied various cross-lingual transfer models trained with data available in the UD. Evaluating on the AfricaPOS dataset, we show that choosing the best transfer language(s) in both single-source and multi-source setups greatly improves the POS tagging performance of the target languages, in particular when combined with parameter-fine-tuning methods. Crucially, transferring knowledge from a language that matches the language family and morphosyntactic properties seems to be more effective for POS tagging in unseen languages.
+
+HomePage: https://github.com/masakhane-io/masakhane-pos
+
+### Citation
+
+```
+@inproceedings{dione-etal-2023-masakhapos,
+    title = "{M}asakha{POS}: Part-of-Speech Tagging for Typologically Diverse {A}frican languages",
+    author = "Dione, Cheikh M. Bamba  and
+      Adelani, David Ifeoluwa  and
+      Nabende, Peter  and
+      Alabi, Jesujoba  and
+      Sindane, Thapelo  and
+      Buzaaba, Happy  and
+      Muhammad, Shamsuddeen Hassan  and
+      Emezue, Chris Chinenye  and
+      Ogayo, Perez  and
+      Aremu, Anuoluwapo  and
+      Gitau, Catherine  and
+      Mbaye, Derguene  and
+      Mukiibi, Jonathan  and
+      Sibanda, Blessing  and
+      Dossou, Bonaventure F. P.  and
+      Bukula, Andiswa  and
+      Mabuya, Rooweither  and
+      Tapo, Allahsera Auguste  and
+      Munkoh-Buabeng, Edwin  and
+      Memdjokam Koagne, Victoire  and
+      Ouoba Kabore, Fatoumata  and
+      Taylor, Amelia  and
+      Kalipe, Godson  and
+      Macucwa, Tebogo  and
+      Marivate, Vukosi  and
+      Gwadabe, Tajuddeen  and
+      Elvis, Mboning Tchiaze  and
+      Onyenwe, Ikechukwu  and
+      Atindogbe, Gratien  and
+      Adelani, Tolulope  and
+      Akinade, Idris  and
+      Samuel, Olanrewaju  and
+      Nahimana, Marien  and
+      Musabeyezu, Th{\'e}og{\`e}ne  and
+      Niyomutabazi, Emile  and
+      Chimhenga, Ester  and
+      Gotosa, Kudzai  and
+      Mizha, Patrick  and
+      Agbolo, Apelete  and
+      Traore, Seydou  and
+      Uchechukwu, Chinedu  and
+      Yusuf, Aliyu  and
+      Abdullahi, Muhammad  and
+      Klakow, Dietrich",
+    editor = "Rogers, Anna  and
+      Boyd-Graber, Jordan  and
+      Okazaki, Naoaki",
+    booktitle = "Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
+    month = jul,
+    year = "2023",
+    address = "Toronto, Canada",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2023.acl-long.609/",
+    doi = "10.18653/v1/2023.acl-long.609",
+    pages = "10883--10900",
+    abstract = "In this paper, we present AfricaPOS, the largest part-of-speech (POS) dataset for 20 typologically diverse African languages. We discuss the challenges in annotating POS for these languages using the universal dependencies (UD) guidelines. We conducted extensive POS baseline experiments using both conditional random field and several multilingual pre-trained language models. We applied various cross-lingual transfer models trained with data available in the UD. Evaluating on the AfricaPOS dataset, we show that choosing the best transfer language(s) in both single-source and multi-source setups greatly improves the POS tagging performance of the target languages, in particular when combined with parameter-fine-tuning methods. Crucially, transferring knowledge from a language that matches the language family and morphosyntactic properties seems to be more effective for POS tagging in unseen languages."
+}
+```

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/gen_utils.py

@@ -0,0 +1,151 @@
+import argparse
+import os
+
+import yaml
+
+
+class FunctionTag:
+    def __init__(self, value):
+        self.value = value
+
+
+def prompt_func(mode, lang):
+    prompt_map = {
+        "prompt_1": "Please provide the POS tags for each word in the input sentence. The input will be a list of "
+        "words in the sentence. The output format should be a list of tuples, where each tuple consists of "
+        "a word from the input text and its corresponding POS tag label from the tag label set: ['ADJ', "
+        "'ADP', 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', "
+        "'SCONJ', 'SYM', 'VERB', 'X']. \nYour response should include only a list of tuples, in the order "
+        "that the words appear in the input sentence, including punctuations, with each tuple containing the corresponding POS tag "
+        "label for a word. \n\nSentence: {{tokens}} \nOutput: ",
+        "prompt_2": f"You are an expert in tagging words and sentences in {lang} with the right POS tag. "
+        f"\n\nPlease provide the POS tags for each word in the {lang} sentence. The input is a list of words in"
+        " the sentence. POS tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', "
+        "'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X']. The output format should "
+        "be a list of tuples, where each tuple consists of a word from the input text and its corresponding"
+        " POS tag label from the POS tag label set provided\nYour response should include only a list of "
+        "tuples, in the order that the words appear in the input sentence, including punctuations, with each tuple containing the "
+        "corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: ",
+        "prompt_3": f"Acting as a {lang} linguist and without making any corrections or changes to the text, perform a part of "
+        "speech (POS) analysis of the sentences using the following POS tag label annotation ['ADJ', "
+        "'ADP', 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', "
+        "'SCONJ', 'SYM', 'VERB', 'X']. The input will be a list of words in the sentence. The output format should "
+        "be a list of tuples, where each tuple consists of a word from the input text and its corresponding"
+        " POS tag label from the POS tag label set provided\nYour response should include only a list of "
+        "tuples, in the order that the words appear in the input sentence, including punctuations, with each tuple containing the "
+        "corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: ",
+        "prompt_4": "Annotate each word in the provided sentence with the appropriate POS tag. The annotation "
+        "list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', "
+        "'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X']. The input sentence will be a list of words"
+        " in the sentence. The output format should "
+        "be a list of tuples, where each tuple consists of a word from the input text and its corresponding"
+        " POS tag label from the POS tag label set provided\nYour response should include only a list of "
+        "tuples, in the order that the words appear in the input sentence, including punctuations, with each tuple containing the "
+        "corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: ",
+        "prompt_5": "Given the following sentence, identify the part of speech (POS) for each word. Use the following "
+        "POS tag set: \nNOUN: Noun (person, place, thing), \nVERB: Verb (action, state), "
+        "\nADJ: Adjective (describes a noun), \nADV: Adverb (modifies a verb, adjective, or adverb), "
+        "\nPRON: Pronoun (replaces a noun), \nDET: Determiner (introduces a noun), "
+        "\nADP: Adposition (preposition or postposition), \nCCONJ: Conjunction (connects words, phrases, clauses)"
+        "\nPUNCT: Punctuation, \nPROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), "
+        "\nSCONJ: Subordinating conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, "
+        "\nNUM: Numeral, \nX: others. The output format should "
+        "be a list of tuples, where each tuple consists of a word from the input text and its corresponding"
+        " POS tag label key only from the POS tag set provided\nYour response should include only a list of "
+        "tuples, in the order that the words appear in the input sentence, including punctuations, with each tuple containing the "
+        "corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: ",
+    }
+    return prompt_map[mode]
+
+
+def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None:
+    """
+    Generate a yaml file for each language.
+
+    :param output_dir: The directory to output the files to.
+    :param overwrite: Whether to overwrite files if they already exist.
+    """
+    err = []
+    languages = {
+        "bam": "Bambara",
+        "bbj": "Ghomala",
+        "ewe": "Ewe",
+        "fon": "Fon",
+        "hau": "Hausa",
+        "ibo": "Igbo",
+        "kin": "Kinyarwanda",
+        "lug": "Luganda",
+        "luo": "Dholuo",
+        "mos": "Mossi",
+        "nya": "Chichewa",
+        "pcm": "Nigerian Pidgin",
+        "sna": "chiShona",
+        "swa": "Kiswahili",
+        "tsn": "Setswana",
+        "twi": "Twi",
+        "wol": "Wolof",
+        "xho": "isiXhosa",
+        "yor": "Yoruba",
+        "zul": "isiZulu",
+    }
+
+    for lang in languages.keys():
+        try:
+            file_name = f"masakhapos_{lang}.yaml"
+            task_name = f"masakhapos_{lang}_{mode}"
+            yaml_template = "masakhapos_yaml"
+            yaml_details = {
+                "include": yaml_template,
+                "task": task_name,
+                "dataset_name": lang,
+                "doc_to_text": prompt_func(mode, languages[lang]),
+            }
+            os.makedirs(f"{output_dir}/{mode}", exist_ok=True)
+            with open(
+                f"{output_dir}/{mode}/{file_name}",
+                "w" if overwrite else "x",
+                encoding="utf8",
+            ) as f:
+                f.write("# Generated by utils.py\n")
+                yaml.dump(
+                    yaml_details,
+                    f,
+                    allow_unicode=True,
+                )
+        except FileExistsError:
+            err.append(file_name)
+
+    if len(err) > 0:
+        raise FileExistsError(
+            "Files were not created because they already exist (use --overwrite flag):"
+            f" {', '.join(err)}"
+        )
+
+
+def main() -> None:
+    """Parse CLI args and generate language-specific yaml files."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--overwrite",
+        default=True,
+        action="store_true",
+        help="Overwrite files if they already exist",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default="./",
+        help="Directory to write yaml files to",
+    )
+    parser.add_argument(
+        "--mode",
+        default="prompt_1",
+        choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"],
+        help="Prompt number",
+    )
+    args = parser.parse_args()
+
+    gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode)
+
+
+if __name__ == "__main__":
+    main()

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_bbj.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: bbj
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_bbj_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_ewe.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: ewe
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_ewe_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_fon.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: fon
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_fon_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_hau.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: hau
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_hau_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_ibo.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_ibo_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_kin.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: kin
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_kin_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_lug.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: lug
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_lug_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_luo.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: luo
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_luo_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_mos.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: mos
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_mos_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_nya.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: nya
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_nya_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_pcm.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: pcm
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_pcm_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_sna.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: sna
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_sna_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_swa.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: swa
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_swa_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_tsn.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: tsn
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_tsn_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_twi.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: twi
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_twi_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_wol.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: wol
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_wol_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_xho.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: xho
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_xho_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_yaml

@@ -0,0 +1,32 @@
+tag:
+- masakhapos_tasks
+- masakhapos_prompt_1
+dataset_path: masakhane/masakhapos
+dataset_name: null
+dataset_kwargs: {trust_remote_code: True}
+output_type: generate_until
+generation_kwargs:
+  do_sample: false
+  until:
+  - </s>
+  - <|im_end|>
+validation_split: validation
+test_split: test
+fewshot_split: train
+doc_to_target: !function utils.doc_to_target
+should_decontaminate: true
+doc_to_decontamination_query: "Sentence: {{token}}\nOutput:"
+filter_list:
+  - filter:
+    - function: regex_pos
+    name: flexible-extract
+metric_list:
+  - metric: acc
+    aggregation: !function utils.acc_score
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+      - ","
+metadata:
+  version: 1.0

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_yor.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: yor
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_yor_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_zul.yaml

@@ -0,0 +1,13 @@
+# Generated by utils.py
+dataset_name: zul
+doc_to_text: "Please provide the POS tags for each word in the input sentence. The\
+  \ input will be a list of words in the sentence. The output format should be a list\
+  \ of tuples, where each tuple consists of a word from the input text and its corresponding\
+  \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\
+  \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\
+  \ 'X']. \nYour response should include only a list of tuples, in the order that\
+  \ the words appear in the input sentence, including punctuations, with each tuple\
+  \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\
+  \ \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_zul_prompt_1

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_1/utils.py

@@ -0,0 +1,55 @@
+from itertools import chain
+
+from sklearn.metrics import accuracy_score
+
+from lm_eval.utils import weighted_f1_score
+
+
+def doc_to_target(doc):
+    pos_tag_map = {
+        0: "NOUN",
+        1: "PUNCT",
+        2: "ADP",
+        3: "NUM",
+        4: "SYM",
+        5: "SCONJ",
+        6: "ADJ",
+        7: "PART",
+        8: "DET",
+        9: "CCONJ",
+        10: "PROPN",
+        11: "PRON",
+        12: "X",
+        13: "_",
+        14: "ADV",
+        15: "INTJ",
+        16: "VERB",
+        17: "AUX",
+    }
+    return [pos_tag_map[tag] for tag in doc["upos"]]
+
+
+def acc_score(items):
+    unzipped_list = list(zip(*items))
+
+    golds, preds = unzipped_list[0], unzipped_list[1]
+
+    # Flatten preds' inner lists
+    flattened_preds = [list(chain.from_iterable(p)) for p in preds]
+
+    # Calculate the accuracy for each gold-pred pair
+    accuracy_scores = []
+    for gold, pred in zip(golds, flattened_preds):
+        # Ensure both lists are of the same length, otherwise truncate to match
+        min_length = min(len(gold), len(pred))
+        gold = gold[:min_length]
+        pred = pred[:min_length]
+
+        # Calculate accuracy for the current pair and add to the list
+        accuracy = accuracy_score(gold, pred)
+        accuracy_scores.append(accuracy)
+
+    mean_accuracy = (
+        sum(accuracy_scores) / len(accuracy_scores) if accuracy_scores else 0
+    )
+    return mean_accuracy

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_bam.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: bam
+doc_to_text: "You are an expert in tagging words and sentences in Bambara with the\
+  \ right POS tag. \n\nPlease provide the POS tags for each word in the Bambara sentence.\
+  \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\
+  \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\
+  \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\
+  \ each tuple consists of a word from the input text and its corresponding POS tag\
+  \ label from the POS tag label set provided\nYour response should include only a\
+  \ list of tuples, in the order that the words appear in the input sentence, including\
+  \ punctuations, with each tuple containing the corresponding POS tag label for a\
+  \ word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_bam_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_bbj.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: bbj
+doc_to_text: "You are an expert in tagging words and sentences in Ghomala with the\
+  \ right POS tag. \n\nPlease provide the POS tags for each word in the Ghomala sentence.\
+  \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\
+  \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\
+  \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\
+  \ each tuple consists of a word from the input text and its corresponding POS tag\
+  \ label from the POS tag label set provided\nYour response should include only a\
+  \ list of tuples, in the order that the words appear in the input sentence, including\
+  \ punctuations, with each tuple containing the corresponding POS tag label for a\
+  \ word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_bbj_prompt_2

lm-evaluation-harness/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_ewe.yaml

@@ -0,0 +1,14 @@
+# Generated by utils.py
+dataset_name: ewe
+doc_to_text: "You are an expert in tagging words and sentences in Ewe with the right\
+  \ POS tag. \n\nPlease provide the POS tags for each word in the Ewe sentence. The\
+  \ input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP', 'ADV',\
+  \ 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\
+  \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\
+  \ each tuple consists of a word from the input text and its corresponding POS tag\
+  \ label from the POS tag label set provided\nYour response should include only a\
+  \ list of tuples, in the order that the words appear in the input sentence, including\
+  \ punctuations, with each tuple containing the corresponding POS tag label for a\
+  \ word. \n\nSentence: {{tokens}} \nOutput: "
+include: masakhapos_yaml
+task: masakhapos_ewe_prompt_2