DA-ctrl-bot / special_tokens_map.json
Akhmadjonov Mumtozbek
add tokenizer
6f403cb
raw
history blame contribute delete
450 Bytes
{"bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "pad_token": "[PAD]", "additional_special_tokens": ["[DA_1]", "[DA_2]", "[DA_3]", "[DA_4]"]}