tokenizer_test6 / special_tokens_map.json
ljcamargo's picture
Upload folder using huggingface_hub
4cd5350 verified
Raw
History Blame Contribute Delete
5.84 kB
{
"pad_token": "<|pad|>",
"bos_token": "<|startoftext|>",
"eos_token": "<|endoftext|>",
"unk_token": "<|unk|>",
"additional_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|tool_call_start|>",
"<|tool_call_end|>",
"<|tool_response_start|>",
"<|tool_response_end|>",
"<|reasoning_start|>",
"<|reasoning_end|>",
"<|fim_prefix|>",
"<|fim_middle|>",
"<|fim_suffix|>",
"<|fim_pad|>",
"<|repo_name|>",
"<|file_sep|>",
"<|lang:agu|>",
"<|lang:all|>",
"<|lang:am1*|>",
"<|lang:amu|>",
"<|lang:amu0|>",
"<|lang:azg|>",
"<|lang:azn|>",
"<|lang:azn0|>",
"<|lang:azz|>",
"<|lang:cac|>",
"<|lang:cak|>",
"<|lang:cco|>",
"<|lang:cco0|>",
"<|lang:chd|>",
"<|lang:chf|>",
"<|lang:chf0|>",
"<|lang:chj|>",
"<|lang:chq|>",
"<|lang:chz|>",
"<|lang:cle|>",
"<|lang:cnl|>",
"<|lang:cnt|>",
"<|lang:cok|>",
"<|lang:coz|>",
"<|lang:cpa|>",
"<|lang:crn|>",
"<|lang:cso|>",
"<|lang:ct2*|>",
"<|lang:cta|>",
"<|lang:cte|>",
"<|lang:ctl|>",
"<|lang:ctp|>",
"<|lang:ctp0|>",
"<|lang:ctu|>",
"<|lang:cuc|>",
"<|lang:cut|>",
"<|lang:cux|>",
"<|lang:de|>",
"<|lang:dih|>",
"<|lang:en-US|>",
"<|lang:eng|>",
"<|lang:es-MX|>",
"<|lang:esp|>",
"<|lang:fr|>",
"<|lang:hch|>",
"<|lang:hu2*|>",
"<|lang:hus|>",
"<|lang:huv|>",
"<|lang:hve|>",
"<|lang:hve0|>",
"<|lang:ixc|>",
"<|lang:jac|>",
"<|lang:jmx|>",
"<|lang:kak|>",
"<|lang:kek|>",
"<|lang:kic|>",
"<|lang:klb|>",
"<|lang:knj|>",
"<|lang:knja|>",
"<|lang:ku1*|>",
"<|lang:lac|>",
"<|lang:maa|>",
"<|lang:maj|>",
"<|lang:mam|>",
"<|lang:maq|>",
"<|lang:mat|>",
"<|lang:mau|>",
"<|lang:maz|>",
"<|lang:maz0|>",
"<|lang:mbz|>",
"<|lang:mbzl|>",
"<|lang:mco|>",
"<|lang:mco0|>",
"<|lang:meh|>",
"<|lang:mfy|>",
"<|lang:mhc|>",
"<|lang:mib|>",
"<|lang:mie|>",
"<|lang:mig|>",
"<|lang:mih|>",
"<|lang:mil|>",
"<|lang:mim|>",
"<|lang:mim0|>",
"<|lang:mio|>",
"<|lang:mir|>",
"<|lang:mit|>",
"<|lang:mix|>",
"<|lang:miy|>",
"<|lang:miz|>",
"<|lang:mjc|>",
"<|lang:mks|>",
"<|lang:ml2*|>",
"<|lang:ml8*|>",
"<|lang:mmc|>",
"<|lang:mop|>",
"<|lang:mpm|>",
"<|lang:mto|>",
"<|lang:mxb|>",
"<|lang:mxp|>",
"<|lang:mxq|>",
"<|lang:mxt|>",
"<|lang:mxv|>",
"<|lang:mxv0|>",
"<|lang:mxy|>",
"<|lang:mza|>",
"<|lang:mzi|>",
"<|lang:mzia|>",
"<|lang:naz|>",
"<|lang:nch|>",
"<|lang:ncj|>",
"<|lang:ncl|>",
"<|lang:ncx|>",
"<|lang:neq|>",
"<|lang:ngu|>",
"<|lang:nhe|>",
"<|lang:nhg|>",
"<|lang:nhi|>",
"<|lang:nhm|>",
"<|lang:nhn|>",
"<|lang:nhw|>",
"<|lang:nhx|>",
"<|lang:nhy|>",
"<|lang:nlv|>",
"<|lang:nlv0|>",
"<|lang:npl|>",
"<|lang:nsu|>",
"<|lang:ntp|>",
"<|lang:ocu|>",
"<|lang:ood|>",
"<|lang:opt|>",
"<|lang:ote|>",
"<|lang:otm|>",
"<|lang:otn|>",
"<|lang:otq|>",
"<|lang:ots|>",
"<|lang:ott|>",
"<|lang:pbs|>",
"<|lang:pei|>",
"<|lang:pia|>",
"<|lang:plo|>",
"<|lang:pls|>",
"<|lang:pmq|>",
"<|lang:pmz|>",
"<|lang:poe|>",
"<|lang:poi|>",
"<|lang:poi0|>",
"<|lang:pos|>",
"<|lang:ppi|>",
"<|lang:pps|>",
"<|lang:pt-BR|>",
"<|lang:pua|>",
"<|lang:quc|>",
"<|lang:sei|>",
"<|lang:spa|>",
"<|lang:st1*|>",
"<|lang:stp|>",
"<|lang:tac|>",
"<|lang:tar|>",
"<|lang:tar0|>",
"<|lang:tcf|>",
"<|lang:tee|>",
"<|lang:thh|>",
"<|lang:tku|>",
"<|lang:tl1*|>",
"<|lang:tla|>",
"<|lang:to1*|>",
"<|lang:toc|>",
"<|lang:toj|>",
"<|lang:too|>",
"<|lang:top|>",
"<|lang:tos|>",
"<|lang:tpl|>",
"<|lang:tpp|>",
"<|lang:tpt|>",
"<|lang:tpx|>",
"<|lang:tpx0|>",
"<|lang:tqt|>",
"<|lang:tr3*|>",
"<|lang:trc|>",
"<|lang:trs|>",
"<|lang:trs0|>",
"<|lang:ts4*|>",
"<|lang:ts9*|>",
"<|lang:tsz|>",
"<|lang:ttc|>",
"<|lang:tzh|>",
"<|lang:tzo|>",
"<|lang:unk|>",
"<|lang:var|>",
"<|lang:vmj|>",
"<|lang:vmm|>",
"<|lang:vmp|>",
"<|lang:vmy|>",
"<|lang:vmy0|>",
"<|lang:vmz|>",
"<|lang:wa1*|>",
"<|lang:xta|>",
"<|lang:xtd|>",
"<|lang:xti|>",
"<|lang:xtj|>",
"<|lang:xtm|>",
"<|lang:xtn|>",
"<|lang:yaq|>",
"<|lang:yua|>",
"<|lang:zaa|>",
"<|lang:zab|>",
"<|lang:zac|>",
"<|lang:zad|>",
"<|lang:zae|>",
"<|lang:zai|>",
"<|lang:zam|>",
"<|lang:zao|>",
"<|lang:zaq|>",
"<|lang:zaq0|>",
"<|lang:zar|>",
"<|lang:zas|>",
"<|lang:zav|>",
"<|lang:zaw|>",
"<|lang:zca|>",
"<|lang:zl3*|>",
"<|lang:zoc|>",
"<|lang:zoh|>",
"<|lang:zoh0|>",
"<|lang:zoq|>",
"<|lang:zoqa|>",
"<|lang:zor|>",
"<|lang:zos|>",
"<|lang:zpb|>",
"<|lang:zpc|>",
"<|lang:zpd|>",
"<|lang:zpf|>",
"<|lang:zpg|>",
"<|lang:zpi|>",
"<|lang:zpl|>",
"<|lang:zpm|>",
"<|lang:zpn|>",
"<|lang:zpo|>",
"<|lang:zpq|>",
"<|lang:zpr|>",
"<|lang:zpt|>",
"<|lang:zpu|>",
"<|lang:zpv|>",
"<|lang:zpz|>",
"<|lang:zsr|>",
"<|lang:ztg|>",
"<|lang:ztp|>",
"<|lang:ztq|>",
"<|lang:zts|>",
"<|reserved_0|>",
"<|reserved_1|>",
"<|reserved_2|>",
"<|reserved_3|>",
"<|reserved_4|>",
"<|reserved_5|>",
"<|reserved_6|>",
"<|reserved_7|>",
"<|reserved_8|>",
"<|reserved_9|>",
"<|reserved_10|>",
"<|reserved_11|>",
"<|reserved_12|>",
"<|reserved_13|>",
"<|reserved_14|>",
"<|reserved_15|>"
]
}