datasets.py

"""All the official datasets in the benchmark."""

from pydantic import BaseModel

from languages import *
from tasks import *


class Dataset(BaseModel):
    """Class to hold dataset information."""

    name: str
    language: Language
    task: Task

    def __hash__(self):
        return hash(self.name)


ALBANIAN_DATASETS = [
    Dataset(name="mms-sq", language=ALBANIAN, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-sq", language=ALBANIAN, task=GRAMMAR),
    Dataset(name="wikiann-sq", language=ALBANIAN, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-sq", language=ALBANIAN, task=READING_COMPREHENSION),
    Dataset(name="lr-sum-sq", language=ALBANIAN, task=SUMMARISATION),
    Dataset(name="global-mmlu-lite-sq", language=ALBANIAN, task=KNOWLEDGE),
    Dataset(name="winogrande-sq", language=ALBANIAN, task=COMMON_SENSE_REASONING),
]

BELARUSIAN_DATASETS = [
    Dataset(name="besls", language=BELARUSIAN, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-be", language=BELARUSIAN, task=GRAMMAR),
    Dataset(name="wikiann-be", language=BELARUSIAN, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-be", language=BELARUSIAN, task=READING_COMPREHENSION),
    Dataset(name="be-wsc", language=BELARUSIAN, task=COMMON_SENSE_REASONING),
]

BOSNIAN_DATASETS = [
    Dataset(name="mms-bs", language=BOSNIAN, task=TEXT_CLASSIFICATION),
    Dataset(name="wikiann-bs", language=BOSNIAN, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-bs", language=BOSNIAN, task=READING_COMPREHENSION),
    Dataset(name="lr-sum-bs", language=BOSNIAN, task=SUMMARISATION),
]

BULGARIAN_DATASETS = [
    Dataset(name="bgt", language=BULGARIAN, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-bg", language=BULGARIAN, task=GRAMMAR),
    Dataset(name="bner", language=BULGARIAN, task=INFORMATION_EXTRACTION),
    Dataset(name="bgquad", language=BULGARIAN, task=READING_COMPREHENSION),
    Dataset(name="cnn-dailymail-bg", language=BULGARIAN, task=SUMMARISATION),
    Dataset(name="mmlu-bg", language=BULGARIAN, task=KNOWLEDGE),
    Dataset(name="winogrande-bg", language=BULGARIAN, task=COMMON_SENSE_REASONING),
]

CATALAN_DATASETS = [
    Dataset(name="guia-cat", language=CATALAN, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-ca", language=CATALAN, task=GRAMMAR),
    Dataset(name="wikiann-ca", language=CATALAN, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-ca", language=CATALAN, task=READING_COMPREHENSION),
    Dataset(name="dacsa-ca", language=CATALAN, task=SUMMARISATION),
    Dataset(name="mmlu-ca", language=CATALAN, task=KNOWLEDGE),
    Dataset(name="winogrande-ca", language=CATALAN, task=COMMON_SENSE_REASONING),
]

CROATIAN_DATASETS = [
    Dataset(name="mms-hr", language=CROATIAN, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-hr", language=CROATIAN, task=GRAMMAR),
    Dataset(name="wikiann-hr", language=CROATIAN, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-hr", language=CROATIAN, task=READING_COMPREHENSION),
    Dataset(name="mmlu-hr", language=CROATIAN, task=KNOWLEDGE),
    Dataset(name="winogrande-hr", language=CROATIAN, task=COMMON_SENSE_REASONING),
]

CZECH_DATASETS = [
    Dataset(name="csfd-sentiment", language=CZECH, task=TEXT_CLASSIFICATION),
    Dataset(name="cs-gec", language=CZECH, task=GRAMMAR),
    Dataset(name="poner", language=CZECH, task=INFORMATION_EXTRACTION),
    Dataset(name="sqad", language=CZECH, task=READING_COMPREHENSION),
    Dataset(name="czech-news", language=CZECH, task=SUMMARISATION),
    Dataset(name="umimeto-qa", language=CZECH, task=KNOWLEDGE),
    Dataset(name="hellaswag-cs", language=CZECH, task=COMMON_SENSE_REASONING),
]

DANISH_DATASETS = [
    Dataset(name="angry-tweets", language=DANISH, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-da", language=DANISH, task=GRAMMAR),
    Dataset(name="dansk", language=DANISH, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-da", language=DANISH, task=READING_COMPREHENSION),
    Dataset(name="nordjylland-news", language=DANISH, task=SUMMARISATION),
    Dataset(name="danske-talemaader", language=DANISH, task=KNOWLEDGE),
    Dataset(name="danish-citizen-tests", language=DANISH, task=KNOWLEDGE),
    Dataset(name="hellaswag-da", language=DANISH, task=COMMON_SENSE_REASONING),
]

DUTCH_DATASETS = [
    Dataset(name="dutch-social", language=DUTCH, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-nl", language=DUTCH, task=GRAMMAR),
    Dataset(name="conll-nl", language=DUTCH, task=INFORMATION_EXTRACTION),
    Dataset(name="squad-nl", language=DUTCH, task=READING_COMPREHENSION),
    Dataset(name="wiki-lingua-nl", language=DUTCH, task=SUMMARISATION),
    Dataset(name="mmlu-nl", language=DUTCH, task=KNOWLEDGE),
    Dataset(name="hellaswag-nl", language=DUTCH, task=COMMON_SENSE_REASONING),
]

ENGLISH_DATASETS = [
    Dataset(name="sst5", language=ENGLISH, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-en", language=ENGLISH, task=GRAMMAR),
    Dataset(name="conll-en", language=ENGLISH, task=INFORMATION_EXTRACTION),
    Dataset(name="squad", language=ENGLISH, task=READING_COMPREHENSION),
    Dataset(name="cnn-dailymail", language=ENGLISH, task=SUMMARISATION),
    Dataset(name="life-in-the-uk", language=ENGLISH, task=KNOWLEDGE),
    Dataset(name="hellaswag", language=ENGLISH, task=COMMON_SENSE_REASONING),
]

ESTONIAN_DATASETS = [
    Dataset(name="estonian-valence", language=ESTONIAN, task=TEXT_CLASSIFICATION),
    Dataset(name="grammar-et", language=ESTONIAN, task=GRAMMAR),
    Dataset(name="estner", language=ESTONIAN, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-et", language=ESTONIAN, task=READING_COMPREHENSION),
    Dataset(name="err-news", language=ESTONIAN, task=SUMMARISATION),
    Dataset(name="trivia-et", language=ESTONIAN, task=KNOWLEDGE),
    Dataset(name="winogrande-et", language=ESTONIAN, task=COMMON_SENSE_REASONING),
]

FAROESE_DATASETS = [
    Dataset(name="fosent", language=FAROESE, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-fo", language=FAROESE, task=GRAMMAR),
    Dataset(name="fone", language=FAROESE, task=INFORMATION_EXTRACTION),
    Dataset(name="foqa", language=FAROESE, task=READING_COMPREHENSION),
]

FINNISH_DATASETS = [
    Dataset(name="scandisent-fi", language=FINNISH, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-fi", language=FINNISH, task=GRAMMAR),
    Dataset(name="turku-ner-fi", language=FINNISH, task=INFORMATION_EXTRACTION),
    Dataset(name="tydiqa-fi", language=FINNISH, task=READING_COMPREHENSION),
    Dataset(name="xlsum-fi", language=FINNISH, task=SUMMARISATION),
    Dataset(name="hellaswag-fi", language=FINNISH, task=COMMON_SENSE_REASONING),
]

FRENCH_DATASETS = [
    Dataset(name="allocine", language=FRENCH, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-fr", language=FRENCH, task=GRAMMAR),
    Dataset(name="eltec", language=FRENCH, task=INFORMATION_EXTRACTION),
    Dataset(name="fquad", language=FRENCH, task=READING_COMPREHENSION),
    Dataset(name="orange-sum", language=FRENCH, task=SUMMARISATION),
    Dataset(name="mmlu-fr", language=FRENCH, task=KNOWLEDGE),
    Dataset(name="hellaswag-fr", language=FRENCH, task=COMMON_SENSE_REASONING),
]

GERMAN_DATASETS = [
    Dataset(name="sb10k", language=GERMAN, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-de", language=GERMAN, task=GRAMMAR),
    Dataset(name="germeval", language=GERMAN, task=INFORMATION_EXTRACTION),
    Dataset(name="germanquad", language=GERMAN, task=READING_COMPREHENSION),
    Dataset(name="mlsum", language=GERMAN, task=SUMMARISATION),
    Dataset(name="mmlu-de", language=GERMAN, task=KNOWLEDGE),
    Dataset(name="hellaswag-de", language=GERMAN, task=COMMON_SENSE_REASONING),
]

GREEK_DATASETS = [
    Dataset(name="greek-sa", language=GREEK, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-el", language=GREEK, task=GRAMMAR),
    Dataset(name="elner", language=GREEK, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-el", language=GREEK, task=READING_COMPREHENSION),
    Dataset(name="greek-wikipedia", language=GREEK, task=SUMMARISATION),
    Dataset(name="global-mmlu-el", language=GREEK, task=KNOWLEDGE),
    Dataset(name="winogrande-el", language=GREEK, task=COMMON_SENSE_REASONING),
]

HUNGARIAN_DATASETS = [
    Dataset(name="husst", language=HUNGARIAN, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-hu", language=HUNGARIAN, task=GRAMMAR),
    Dataset(name="szeged-ner", language=HUNGARIAN, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-hu", language=HUNGARIAN, task=READING_COMPREHENSION),
    Dataset(name="hunsum", language=HUNGARIAN, task=SUMMARISATION),
    Dataset(name="mmlu-hu", language=HUNGARIAN, task=KNOWLEDGE),
    Dataset(name="winogrande-hu", language=HUNGARIAN, task=COMMON_SENSE_REASONING),
]

ICELANDIC_DATASETS = [
    Dataset(
        name="hotter-and-colder-sentiment", language=ICELANDIC, task=TEXT_CLASSIFICATION
    ),
    Dataset(name="scala-is", language=ICELANDIC, task=GRAMMAR),
    Dataset(name="mim-gold-ner", language=ICELANDIC, task=INFORMATION_EXTRACTION),
    Dataset(name="nqii", language=ICELANDIC, task=READING_COMPREHENSION),
    Dataset(name="rrn", language=ICELANDIC, task=SUMMARISATION),
    Dataset(name="icelandic-knowledge", language=ICELANDIC, task=KNOWLEDGE),
    Dataset(name="winogrande-is", language=ICELANDIC, task=COMMON_SENSE_REASONING),
]

ITALIAN_DATASETS = [
    Dataset(name="sentipolc16", language=ITALIAN, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-it", language=ITALIAN, task=GRAMMAR),
    Dataset(name="multinerd-it", language=ITALIAN, task=INFORMATION_EXTRACTION),
    Dataset(name="squad-it", language=ITALIAN, task=READING_COMPREHENSION),
    Dataset(name="ilpost-sum", language=ITALIAN, task=SUMMARISATION),
    Dataset(name="mmlu-it", language=ITALIAN, task=KNOWLEDGE),
    Dataset(name="hellaswag-it", language=ITALIAN, task=COMMON_SENSE_REASONING),
]

LATVIAN_DATASETS = [
    Dataset(
        name="latvian-twitter-sentiment", language=LATVIAN, task=TEXT_CLASSIFICATION
    ),
    Dataset(name="scala-lv", language=LATVIAN, task=GRAMMAR),
    Dataset(name="fullstack-ner-lv", language=LATVIAN, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-lv", language=LATVIAN, task=READING_COMPREHENSION),
    Dataset(name="lsm", language=LATVIAN, task=SUMMARISATION),
    Dataset(name="mmlu-lv", language=LATVIAN, task=KNOWLEDGE),
    Dataset(name="copa-lv", language=LATVIAN, task=COMMON_SENSE_REASONING),
]

LITHUANIAN_DATASETS = [
    Dataset(name="atsiliepimai", language=LITHUANIAN, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-lt", language=LITHUANIAN, task=GRAMMAR),
    Dataset(name="wikiann-lt", language=LITHUANIAN, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-lt", language=LITHUANIAN, task=READING_COMPREHENSION),
    Dataset(name="lrytas", language=LITHUANIAN, task=SUMMARISATION),
    Dataset(name="lt-history", language=LITHUANIAN, task=KNOWLEDGE),
    Dataset(name="winogrande-lt", language=LITHUANIAN, task=COMMON_SENSE_REASONING),
]

NORWEGIAN_DATASETS = [
    Dataset(name="norec", language=NORWEGIAN, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-nb", language=NORWEGIAN, task=GRAMMAR),
    Dataset(name="scala-nn", language=NORWEGIAN, task=GRAMMAR),
    Dataset(name="norne-nb", language=NORWEGIAN, task=INFORMATION_EXTRACTION),
    Dataset(name="norne-nn", language=NORWEGIAN, task=INFORMATION_EXTRACTION),
    Dataset(name="norquad", language=NORWEGIAN, task=READING_COMPREHENSION),
    Dataset(name="no-sammendrag", language=NORWEGIAN, task=SUMMARISATION),
    Dataset(name="nrk-quiz-qa", language=NORWEGIAN, task=KNOWLEDGE),
    Dataset(
        name="nor-common-sense-qa", language=NORWEGIAN, task=COMMON_SENSE_REASONING
    ),
]

POLISH_DATASETS = [
    Dataset(name="polemo2", language=POLISH, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-pl", language=POLISH, task=GRAMMAR),
    Dataset(name="kpwr-ner", language=POLISH, task=INFORMATION_EXTRACTION),
    Dataset(name="poquad", language=POLISH, task=READING_COMPREHENSION),
    Dataset(name="psc", language=POLISH, task=SUMMARISATION),
    Dataset(name="llmzszl", language=POLISH, task=KNOWLEDGE),
    Dataset(name="winogrande-pl", language=POLISH, task=COMMON_SENSE_REASONING),
]

PORTUGUESE_DATASETS = [
    Dataset(name="sst2-pt", language=PORTUGUESE, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-pt", language=PORTUGUESE, task=GRAMMAR),
    Dataset(name="harem", language=PORTUGUESE, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-pt", language=PORTUGUESE, task=READING_COMPREHENSION),
    Dataset(name="publico", language=PORTUGUESE, task=SUMMARISATION),
    Dataset(name="mmlu-pt", language=PORTUGUESE, task=KNOWLEDGE),
    Dataset(name="goldenswag-pt", language=PORTUGUESE, task=COMMON_SENSE_REASONING),
]

ROMANIAN_DATASETS = [
    Dataset(name="ro-sent", language=ROMANIAN, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-ro", language=ROMANIAN, task=GRAMMAR),
    Dataset(name="ronec", language=ROMANIAN, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-ro", language=ROMANIAN, task=READING_COMPREHENSION),
    Dataset(name="sumo-ro", language=ROMANIAN, task=SUMMARISATION),
    Dataset(name="global-mmlu-ro", language=ROMANIAN, task=KNOWLEDGE),
    Dataset(name="winogrande-ro", language=ROMANIAN, task=COMMON_SENSE_REASONING),
]

SERBIAN_DATASETS = [
    Dataset(name="mms-sr", language=SERBIAN, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-sr", language=SERBIAN, task=GRAMMAR),
    Dataset(name="uner-sr", language=SERBIAN, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-sr", language=SERBIAN, task=READING_COMPREHENSION),
    Dataset(name="lr-sum-sr", language=SERBIAN, task=SUMMARISATION),
    Dataset(name="mmlu-sr", language=SERBIAN, task=KNOWLEDGE),
    Dataset(name="winogrande-sr", language=SERBIAN, task=COMMON_SENSE_REASONING),
]

SLOVAK_DATASETS = [
    Dataset(name="csfd-sentiment-sk", language=SLOVAK, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-sk", language=SLOVAK, task=GRAMMAR),
    Dataset(name="uner-sk", language=SLOVAK, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-sk", language=SLOVAK, task=READING_COMPREHENSION),
    Dataset(name="mmlu-sk", language=SLOVAK, task=KNOWLEDGE),
    Dataset(name="winogrande-sk", language=SLOVAK, task=COMMON_SENSE_REASONING),
]

SLOVENE_DATASETS = [
    Dataset(name="sentinews", language=SLOVENE, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-sl", language=SLOVENE, task=GRAMMAR),
    Dataset(name="ssj500k-ner", language=SLOVENE, task=INFORMATION_EXTRACTION),
    Dataset(name="multi-wiki-qa-sl", language=SLOVENE, task=READING_COMPREHENSION),
    Dataset(name="mmlu-sl", language=SLOVENE, task=KNOWLEDGE),
    Dataset(name="winogrande-sl", language=SLOVENE, task=COMMON_SENSE_REASONING),
]

SPANISH_DATASETS = [
    Dataset(name="sentiment-headlines-es", language=SPANISH, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-es", language=SPANISH, task=GRAMMAR),
    Dataset(name="conll-es", language=SPANISH, task=INFORMATION_EXTRACTION),
    Dataset(name="mlqa-es", language=SPANISH, task=READING_COMPREHENSION),
    Dataset(name="mlsum-es", language=SPANISH, task=SUMMARISATION),
    Dataset(name="mmlu-es", language=SPANISH, task=KNOWLEDGE),
    Dataset(name="hellaswag-es", language=SPANISH, task=COMMON_SENSE_REASONING),
]

SWEDISH_DATASETS = [
    Dataset(name="swerec", language=SWEDISH, task=TEXT_CLASSIFICATION),
    Dataset(name="scala-sv", language=SWEDISH, task=GRAMMAR),
    Dataset(name="suc3", language=SWEDISH, task=INFORMATION_EXTRACTION),
    Dataset(name="scandiqa-sv", language=SWEDISH, task=READING_COMPREHENSION),
    Dataset(name="swedn", language=SWEDISH, task=SUMMARISATION),
    Dataset(name="mmlu-sv", language=SWEDISH, task=KNOWLEDGE),
    Dataset(name="hellaswag-sv", language=SWEDISH, task=COMMON_SENSE_REASONING),
]

DATASETS = (
    ALBANIAN_DATASETS
    + BELARUSIAN_DATASETS
    + BOSNIAN_DATASETS
    + BULGARIAN_DATASETS
    + CATALAN_DATASETS
    + CROATIAN_DATASETS
    + CZECH_DATASETS
    + DANISH_DATASETS
    + DUTCH_DATASETS
    + ENGLISH_DATASETS
    + ESTONIAN_DATASETS
    + FAROESE_DATASETS
    + FINNISH_DATASETS
    + FRENCH_DATASETS
    + GERMAN_DATASETS
    + GREEK_DATASETS
    + HUNGARIAN_DATASETS
    + ICELANDIC_DATASETS
    + ITALIAN_DATASETS
    + LATVIAN_DATASETS
    + LITHUANIAN_DATASETS
    + NORWEGIAN_DATASETS
    + POLISH_DATASETS
    + PORTUGUESE_DATASETS
    + ROMANIAN_DATASETS
    + SERBIAN_DATASETS
    + SLOVAK_DATASETS
    + SLOVENE_DATASETS
    + SPANISH_DATASETS
    + SWEDISH_DATASETS
)