| { |
| "added_tokens_decoder": { |
| "0": { |
| "content": "<s>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "1": { |
| "content": "<pad>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "2": { |
| "content": "</s>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "3": { |
| "content": "<unk>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75859": { |
| "content": "<mask>", |
| "lstrip": true, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75860": { |
| "content": "ace_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75861": { |
| "content": "ace_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75862": { |
| "content": "acm_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75863": { |
| "content": "acq_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75864": { |
| "content": "aeb_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75865": { |
| "content": "afr_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75866": { |
| "content": "ajp_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75867": { |
| "content": "aka_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75868": { |
| "content": "als_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75869": { |
| "content": "amh_Ethi", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75870": { |
| "content": "apc_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75871": { |
| "content": "arb_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75872": { |
| "content": "ars_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75873": { |
| "content": "ary_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75874": { |
| "content": "arz_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75875": { |
| "content": "asm_Beng", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75876": { |
| "content": "ast_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75877": { |
| "content": "awa_Deva", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75878": { |
| "content": "ayr_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75879": { |
| "content": "azb_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75880": { |
| "content": "azj_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75881": { |
| "content": "bak_Cyrl", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75882": { |
| "content": "bam_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75883": { |
| "content": "ban_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75884": { |
| "content": "bel_Cyrl", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75885": { |
| "content": "bem_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75886": { |
| "content": "ben_Beng", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75887": { |
| "content": "bho_Deva", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75888": { |
| "content": "bjn_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75889": { |
| "content": "bjn_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75890": { |
| "content": "bod_Tibt", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75891": { |
| "content": "bos_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75892": { |
| "content": "bug_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75893": { |
| "content": "bul_Cyrl", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75894": { |
| "content": "cat_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75895": { |
| "content": "ceb_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75896": { |
| "content": "ces_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75897": { |
| "content": "cjk_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75898": { |
| "content": "ckb_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75899": { |
| "content": "crh_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75900": { |
| "content": "cym_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75901": { |
| "content": "dan_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75902": { |
| "content": "deu_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75903": { |
| "content": "dik_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75904": { |
| "content": "dyu_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75905": { |
| "content": "dzo_Tibt", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75906": { |
| "content": "ell_Grek", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75907": { |
| "content": "eng_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75908": { |
| "content": "epo_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75909": { |
| "content": "est_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75910": { |
| "content": "eus_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75911": { |
| "content": "ewe_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75912": { |
| "content": "fao_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75913": { |
| "content": "fij_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75914": { |
| "content": "fin_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75915": { |
| "content": "fon_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75916": { |
| "content": "fra_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75917": { |
| "content": "fur_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75918": { |
| "content": "fuv_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75919": { |
| "content": "gaz_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75920": { |
| "content": "gla_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75921": { |
| "content": "gle_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75922": { |
| "content": "glg_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75923": { |
| "content": "grn_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75924": { |
| "content": "guj_Gujr", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75925": { |
| "content": "hat_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75926": { |
| "content": "hau_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75927": { |
| "content": "heb_Hebr", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75928": { |
| "content": "hin_Deva", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75929": { |
| "content": "hne_Deva", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75930": { |
| "content": "hrv_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75931": { |
| "content": "hun_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75932": { |
| "content": "hye_Armn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75933": { |
| "content": "ibo_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75934": { |
| "content": "ilo_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75935": { |
| "content": "ind_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75936": { |
| "content": "isl_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75937": { |
| "content": "ita_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75938": { |
| "content": "jav_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75939": { |
| "content": "jpn_Jpan", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75940": { |
| "content": "kab_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75941": { |
| "content": "kac_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75942": { |
| "content": "kam_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75943": { |
| "content": "kan_Knda", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75944": { |
| "content": "kas_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75945": { |
| "content": "kas_Deva", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75946": { |
| "content": "kat_Geor", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75947": { |
| "content": "kaz_Cyrl", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75948": { |
| "content": "kbp_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75949": { |
| "content": "kea_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75950": { |
| "content": "khk_Cyrl", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75951": { |
| "content": "khm_Khmr", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75952": { |
| "content": "kik_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75953": { |
| "content": "kin_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75954": { |
| "content": "kir_Cyrl", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75955": { |
| "content": "kmb_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75956": { |
| "content": "kmr_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75957": { |
| "content": "knc_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75958": { |
| "content": "knc_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75959": { |
| "content": "kon_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75960": { |
| "content": "kor_Hang", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75961": { |
| "content": "lao_Laoo", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75962": { |
| "content": "lij_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75963": { |
| "content": "lim_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75964": { |
| "content": "lin_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75965": { |
| "content": "lit_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75966": { |
| "content": "lmo_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75967": { |
| "content": "ltg_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75968": { |
| "content": "ltz_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75969": { |
| "content": "lua_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75970": { |
| "content": "lug_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75971": { |
| "content": "luo_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75972": { |
| "content": "lus_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75973": { |
| "content": "lvs_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75974": { |
| "content": "mag_Deva", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75975": { |
| "content": "mai_Deva", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75976": { |
| "content": "mal_Mlym", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75977": { |
| "content": "mar_Deva", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75978": { |
| "content": "min_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75979": { |
| "content": "mkd_Cyrl", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75980": { |
| "content": "mlt_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75981": { |
| "content": "mni_Beng", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75982": { |
| "content": "mos_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75983": { |
| "content": "mri_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75984": { |
| "content": "mya_Mymr", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75985": { |
| "content": "nld_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75986": { |
| "content": "nno_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75987": { |
| "content": "nob_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75988": { |
| "content": "npi_Deva", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75989": { |
| "content": "nso_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75990": { |
| "content": "nus_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75991": { |
| "content": "nya_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75992": { |
| "content": "oci_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75993": { |
| "content": "ory_Orya", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75994": { |
| "content": "pag_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75995": { |
| "content": "pan_Guru", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75996": { |
| "content": "pap_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75997": { |
| "content": "pbt_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75998": { |
| "content": "pes_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "75999": { |
| "content": "plt_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76000": { |
| "content": "pol_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76001": { |
| "content": "por_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76002": { |
| "content": "prs_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76003": { |
| "content": "quy_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76004": { |
| "content": "ron_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76005": { |
| "content": "run_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76006": { |
| "content": "rus_Cyrl", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76007": { |
| "content": "sag_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76008": { |
| "content": "san_Deva", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76009": { |
| "content": "sat_Beng", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76010": { |
| "content": "scn_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76011": { |
| "content": "shn_Mymr", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76012": { |
| "content": "sin_Sinh", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76013": { |
| "content": "slk_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76014": { |
| "content": "slv_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76015": { |
| "content": "smo_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76016": { |
| "content": "sna_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76017": { |
| "content": "snd_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76018": { |
| "content": "som_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76019": { |
| "content": "sot_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76020": { |
| "content": "spa_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76021": { |
| "content": "srd_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76022": { |
| "content": "srp_Cyrl", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76023": { |
| "content": "ssw_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76024": { |
| "content": "sun_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76025": { |
| "content": "swe_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76026": { |
| "content": "swh_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76027": { |
| "content": "szl_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76028": { |
| "content": "tam_Taml", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76029": { |
| "content": "taq_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76030": { |
| "content": "taq_Tfng", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76031": { |
| "content": "tat_Cyrl", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76032": { |
| "content": "tel_Telu", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76033": { |
| "content": "tgk_Cyrl", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76034": { |
| "content": "tgl_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76035": { |
| "content": "tha_Thai", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76036": { |
| "content": "tig_Ethi", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76037": { |
| "content": "tir_Ethi", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76038": { |
| "content": "tpi_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76039": { |
| "content": "tsn_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76040": { |
| "content": "tso_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76041": { |
| "content": "tuk_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76042": { |
| "content": "tum_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76043": { |
| "content": "tur_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76044": { |
| "content": "twi_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76045": { |
| "content": "tzm_Tfng", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76046": { |
| "content": "uig_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76047": { |
| "content": "ukr_Cyrl", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76048": { |
| "content": "umb_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76049": { |
| "content": "urd_Arab", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76050": { |
| "content": "uzn_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76051": { |
| "content": "vec_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76052": { |
| "content": "vie_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76053": { |
| "content": "war_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76054": { |
| "content": "wol_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76055": { |
| "content": "xho_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76056": { |
| "content": "ydd_Hebr", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76057": { |
| "content": "yor_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76058": { |
| "content": "yue_Hant", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76059": { |
| "content": "zho_Hans", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76060": { |
| "content": "zho_Hant", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76061": { |
| "content": "zsm_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "76062": { |
| "content": "zul_Latn", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| } |
| }, |
| "additional_special_tokens": [ |
| "ace_Arab", |
| "ace_Latn", |
| "acm_Arab", |
| "acq_Arab", |
| "aeb_Arab", |
| "afr_Latn", |
| "ajp_Arab", |
| "aka_Latn", |
| "als_Latn", |
| "amh_Ethi", |
| "apc_Arab", |
| "arb_Arab", |
| "ars_Arab", |
| "ary_Arab", |
| "arz_Arab", |
| "asm_Beng", |
| "ast_Latn", |
| "awa_Deva", |
| "ayr_Latn", |
| "azb_Arab", |
| "azj_Latn", |
| "bak_Cyrl", |
| "bam_Latn", |
| "ban_Latn", |
| "bel_Cyrl", |
| "bem_Latn", |
| "ben_Beng", |
| "bho_Deva", |
| "bjn_Arab", |
| "bjn_Latn", |
| "bod_Tibt", |
| "bos_Latn", |
| "bug_Latn", |
| "bul_Cyrl", |
| "cat_Latn", |
| "ceb_Latn", |
| "ces_Latn", |
| "cjk_Latn", |
| "ckb_Arab", |
| "crh_Latn", |
| "cym_Latn", |
| "dan_Latn", |
| "deu_Latn", |
| "dik_Latn", |
| "dyu_Latn", |
| "dzo_Tibt", |
| "ell_Grek", |
| "eng_Latn", |
| "epo_Latn", |
| "est_Latn", |
| "eus_Latn", |
| "ewe_Latn", |
| "fao_Latn", |
| "fij_Latn", |
| "fin_Latn", |
| "fon_Latn", |
| "fra_Latn", |
| "fur_Latn", |
| "fuv_Latn", |
| "gaz_Latn", |
| "gla_Latn", |
| "gle_Latn", |
| "glg_Latn", |
| "grn_Latn", |
| "guj_Gujr", |
| "hat_Latn", |
| "hau_Latn", |
| "heb_Hebr", |
| "hin_Deva", |
| "hne_Deva", |
| "hrv_Latn", |
| "hun_Latn", |
| "hye_Armn", |
| "ibo_Latn", |
| "ilo_Latn", |
| "ind_Latn", |
| "isl_Latn", |
| "ita_Latn", |
| "jav_Latn", |
| "jpn_Jpan", |
| "kab_Latn", |
| "kac_Latn", |
| "kam_Latn", |
| "kan_Knda", |
| "kas_Arab", |
| "kas_Deva", |
| "kat_Geor", |
| "kaz_Cyrl", |
| "kbp_Latn", |
| "kea_Latn", |
| "khk_Cyrl", |
| "khm_Khmr", |
| "kik_Latn", |
| "kin_Latn", |
| "kir_Cyrl", |
| "kmb_Latn", |
| "kmr_Latn", |
| "knc_Arab", |
| "knc_Latn", |
| "kon_Latn", |
| "kor_Hang", |
| "lao_Laoo", |
| "lij_Latn", |
| "lim_Latn", |
| "lin_Latn", |
| "lit_Latn", |
| "lmo_Latn", |
| "ltg_Latn", |
| "ltz_Latn", |
| "lua_Latn", |
| "lug_Latn", |
| "luo_Latn", |
| "lus_Latn", |
| "lvs_Latn", |
| "mag_Deva", |
| "mai_Deva", |
| "mal_Mlym", |
| "mar_Deva", |
| "min_Latn", |
| "mkd_Cyrl", |
| "mlt_Latn", |
| "mni_Beng", |
| "mos_Latn", |
| "mri_Latn", |
| "mya_Mymr", |
| "nld_Latn", |
| "nno_Latn", |
| "nob_Latn", |
| "npi_Deva", |
| "nso_Latn", |
| "nus_Latn", |
| "nya_Latn", |
| "oci_Latn", |
| "ory_Orya", |
| "pag_Latn", |
| "pan_Guru", |
| "pap_Latn", |
| "pbt_Arab", |
| "pes_Arab", |
| "plt_Latn", |
| "pol_Latn", |
| "por_Latn", |
| "prs_Arab", |
| "quy_Latn", |
| "ron_Latn", |
| "run_Latn", |
| "rus_Cyrl", |
| "sag_Latn", |
| "san_Deva", |
| "sat_Beng", |
| "scn_Latn", |
| "shn_Mymr", |
| "sin_Sinh", |
| "slk_Latn", |
| "slv_Latn", |
| "smo_Latn", |
| "sna_Latn", |
| "snd_Arab", |
| "som_Latn", |
| "sot_Latn", |
| "spa_Latn", |
| "srd_Latn", |
| "srp_Cyrl", |
| "ssw_Latn", |
| "sun_Latn", |
| "swe_Latn", |
| "swh_Latn", |
| "szl_Latn", |
| "tam_Taml", |
| "taq_Latn", |
| "taq_Tfng", |
| "tat_Cyrl", |
| "tel_Telu", |
| "tgk_Cyrl", |
| "tgl_Latn", |
| "tha_Thai", |
| "tig_Ethi", |
| "tir_Ethi", |
| "tpi_Latn", |
| "tsn_Latn", |
| "tso_Latn", |
| "tuk_Latn", |
| "tum_Latn", |
| "tur_Latn", |
| "twi_Latn", |
| "tzm_Tfng", |
| "uig_Arab", |
| "ukr_Cyrl", |
| "umb_Latn", |
| "urd_Arab", |
| "uzn_Latn", |
| "vec_Latn", |
| "vie_Latn", |
| "war_Latn", |
| "wol_Latn", |
| "xho_Latn", |
| "ydd_Hebr", |
| "yor_Latn", |
| "yue_Hant", |
| "zho_Hans", |
| "zho_Hant", |
| "zsm_Latn", |
| "zul_Latn" |
| ], |
| "bos_token": "<s>", |
| "clean_up_tokenization_spaces": true, |
| "cls_token": "<s>", |
| "eos_token": "</s>", |
| "extra_special_tokens": {}, |
| "legacy_behaviour": false, |
| "mask_token": "<mask>", |
| "model_max_length": 1024, |
| "pad_token": "<pad>", |
| "sep_token": "</s>", |
| "sp_model_kwargs": {}, |
| "src_lang": "tig_Ethi", |
| "tgt_lang": null, |
| "tokenizer_class": "NllbTokenizer", |
| "unk_token": "<unk>" |
| } |
|
|
