WMT24++

This repository contains the human translation and post-edit data for the 55 en->xx language pairs released in the publication WMT24++: Expanding the Language Coverage of WMT24 to 55 Languages & Dialects. If you are interested in the MT/LLM system outputs and automatic metric scores, please see MTME. If you are interested in the images of the source URLs for each document, please see here.

Schema

Each language pair is stored in its own jsonl file. Each row is a serialized JSON object with the following fields:

• lp: The language pair (e.g., "en-de_DE").
• domain: The domain of the source, either "canary", "news", "social", "speech", or "literary".
• document_id: The unique ID that identifies the document the source came from.
• segment_id: The globally unique ID that identifies the segment.
• is_bad_source: A Boolean that indicates whether this source is low quality (e.g., HTML, URLs, emoijs). In the paper, the segments marked as true were removed from the evaluation, and we recommend doing the same.
• source: The English source text.
• target: The post-edit of original_target. We recommend using the post-edit as the default reference.
• original_target: The original reference translation.

Citation

If you use any of the data released in our work, please cite the following paper:

@misc{deutsch2025wmt24expandinglanguagecoverage,
      title={{WMT24++: Expanding the Language Coverage of WMT24 to 55 Languages & Dialects}}, 
      author={Daniel Deutsch and Eleftheria Briakou and Isaac Caswell and Mara Finkelstein and Rebecca Galor and Juraj Juraska and Geza Kovacs and Alison Lui and Ricardo Rei and Jason Riesa and Shruti Rijhwani and Parker Riley and Elizabeth Salesky and Firas Trabelsi and Stephanie Winkler and Biao Zhang and Markus Freitag},
      year={2025},
      eprint={2502.12404},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2502.12404}, 
}

Extensions of this benchmark

• https://huggingface.co/datasets/ZurichNLP/wmt24pp-rm (German → 6 varieties of Romansh)
• https://huggingface.co/datasets/NM-development/wmt24pp-ce (English → Chechen) from Vladislav Grigorian and Abu-Viskhan Umishov

Helpful Python Constants

LANGUAGE_PAIRS = (
    "en-ar_EG", "en-ar_SA", "en-bg_BG", "en-bn_IN", "en-ca_ES", "en-cs_CZ", "en-da_DK", "en-de_DE",
    "en-el_GR", "en-es_MX", "en-et_EE", "en-fa_IR", "en-fi_FI", "en-fil_PH", "en-fr_CA", "en-fr_FR",
    "en-gu_IN", "en-he_IL", "en-hi_IN", "en-hr_HR", "en-hu_HU", "en-id_ID", "en-is_IS", "en-it_IT",
    "en-ja_JP", "en-kn_IN", "en-ko_KR", "en-lt_LT", "en-lv_LV", "en-ml_IN", "en-mr_IN", "en-nl_NL",
    "en-no_NO", "en-pa_IN", "en-pl_PL", "en-pt_BR", "en-pt_PT", "en-ro_RO", "en-ru_RU", "en-sk_SK",
    "en-sl_SI", "en-sr_RS", "en-sv_SE", "en-sw_KE", "en-sw_TZ", "en-ta_IN", "en-te_IN", "en-th_TH",
    "en-tr_TR", "en-uk_UA", "en-ur_PK", "en-vi_VN", "en-zh_CN", "en-zh_TW", "en-zu_ZA",
)

LANGUAGE_BY_CODE = {
    "ar_EG": "Arabic",
    "ar_SA": "Arabic",
    "bg_BG": "Bulgarian",
    "bn_BD": "Bengali",
    "bn_IN": "Bengali",
    "ca_ES": "Catalan",
    "cs_CZ": "Czech",
    "da_DK": "Danish",
    "de_DE": "German",
    "el_GR": "Greek",
    "es_MX": "Spanish",
    "et_EE": "Estonian",
    "fa_IR": "Farsi",
    "fi_FI": "Finnish",
    "fil_PH": "Filipino",
    "fr_CA": "French",
    "fr_FR": "French",
    "gu_IN": "Gujarati",
    "he_IL": "Hebrew",
    "hi_IN": "Hindi",
    "hr_HR": "Croatian",
    "hu_HU": "Hungarian",
    "id_ID": "Indonesian",
    "is_IS": "Icelandic",
    "it_IT": "Italian",
    "ja_JP": "Japanese",
    "kn_IN": "Kannada",
    "ko_KR": "Korean",
    "lt_LT": "Lithuanian",
    "lv_LV": "Latvian",
    "ml_IN": "Malayalam",
    "mr_IN": "Marathi",
    "nl_NL": "Dutch",
    "no_NO": "Norwegian",
    "pa_IN": "Punjabi",
    "pl_PL": "Polish",
    "pt_BR": "Portuguese",
    "pt_PT": "Portuguese",
    "ro_RO": "Romanian",
    "ru_RU": "Russian",
    "sk_SK": "Slovak",
    "sl_SI": "Slovenian",
    "sr_RS": "Serbian",
    "sv_SE": "Swedish",
    "sw_KE": "Swahili",
    "sw_TZ": "Swahili",
    "ta_IN": "Tamil",
    "te_IN": "Telugu",
    "th_TH": "Thai",
    "tr_TR": "Turkish",
    "uk_UA": "Ukrainian",
    "ur_PK": "Urdu",
    "vi_VN": "Vietnamese",
    "zh_CN": "Mandarin",
    "zh_TW": "Mandarin",
    "zu_ZA": "Zulu",
}

REGION_BY_CODE = {
    "ar_EG": "Egypt",
    "ar_SA": "Saudi Arabia",
    "bg_BG": "Bulgaria",
    "bn_BD": "Bangladesh",
    "bn_IN": "India",
    "ca_ES": "Spain",
    "cs_CZ": "Czechia",
    "da_DK": "Denmark",
    "de_DE": "Germany",
    "el_GR": "Greece",
    "es_MX": "Mexico",
    "et_EE": "Estonia",
    "fa_IR": "Iran",
    "fi_FI": "Finland",
    "fil_PH": "Philippines",
    "fr_CA": "Canada",
    "fr_FR": "France",
    "gu_IN": "India",
    "he_IL": "Israel",
    "hi_IN": "India",
    "hr_HR": "Croatia",
    "hu_HU": "Hungary",
    "id_ID": "Indonesia",
    "is_IS": "Iceland",
    "it_IT": "Italy",
    "ja_JP": "Japan",
    "kn_IN": "India",
    "ko_KR": "South Korea",
    "lt_LT": "Lithuania",
    "lv_LV": "Latvia",
    "ml_IN": "India",
    "mr_IN": "India",
    "nl_NL": "Netherlands",
    "no_NO": "Norway",
    "pa_IN": "India",
    "pl_PL": "Poland",
    "pt_BR": "Brazil",
    "pt_PT": "Portugal",
    "ro_RO": "Romania",
    "ru_RU": "Russia",
    "sk_SK": "Slovakia",
    "sl_SI": "Slovenia",
    "sr_RS": "Serbia",
    "sv_SE": "Sweden",
    "sw_KE": "Kenya",
    "sw_TZ": "Tanzania",
    "ta_IN": "India",
    "te_IN": "India",
    "th_TH": "Thailand",
    "tr_TR": "Turkey",
    "uk_UA": "Ukraine",
    "ur_PK": "Pakistan",
    "vi_VN": "Vietnam",
    "zh_CN": "China",
    "zh_TW": "Taiwan",
    "zu_ZA": "South Africa",
}