Transync / special_tokens_map.json
RohanMuralidharan's picture
Upload 10 files
3fa8f3e verified
Raw
History Blame
811 Bytes
{
"eos_token": "</s>",
"unk_token": "<unk>",
"sep_token": "</s>",
"pad_token": "<pad>",
"cls_token": "<s>",
"mask_token": {
"content": "<mask>",
"single_word": false,
"lstrip": true,
"rstrip": false,
"normalized": true
},
"additional_special_tokens": [
"af_ZA", "ar_AR", "az_AZ", "bn_IN", "cs_CZ", "de_DE", "en_XX", "es_XX",
"et_EE", "fa_IR", "fi_FI", "fr_XX", "gl_ES", "gu_IN", "he_IL", "hi_IN",
"hr_HR", "id_ID", "it_IT", "ja_XX", "ka_GE", "kk_KZ", "km_KH", "ko_KR",
"lt_LT", "lv_LV", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "my_MM", "ne_NP",
"nl_XX", "pl_PL", "ps_AF", "pt_XX", "ro_RO", "ru_RU", "si_LK", "sl_SI",
"sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "tr_TR", "uk_UA",
"ur_PK", "vi_VN", "xh_ZA", "zh_CN"
]
}