Upload tokenizer
90b8870 verified | {
|
| "bos_token": {
|
| "content": "<|bos|>",
|
| "lstrip": false,
|
| "normalized": false,
|
| "rstrip": false,
|
| "single_word": false
|
| },
|
| "eos_token": {
|
| "content": "<|eos|>",
|
| "lstrip": false,
|
| "normalized": false,
|
| "rstrip": false,
|
| "single_word": false
|
| },
|
| "mask_token": {
|
| "content": "<|mask|>",
|
| "lstrip": false,
|
| "normalized": false,
|
| "rstrip": false,
|
| "single_word": false
|
| },
|
| "pad_token": {
|
| "content": "<|pad|>",
|
| "lstrip": false,
|
| "normalized": false,
|
| "rstrip": false,
|
| "single_word": false
|
| },
|
| "sep_token": {
|
| "content": "<|sep|>",
|
| "lstrip": false,
|
| "normalized": false,
|
| "rstrip": false,
|
| "single_word": false
|
| },
|
| "unk_token": {
|
| "content": "<|unk|>",
|
| "lstrip": false,
|
| "normalized": false,
|
| "rstrip": false,
|
| "single_word": false
|
| }
|
| }
|
|
|
