Upload tokenizer
4e0c4c9 verified | | { |
| | "cls_token": { |
| | "content": "[CLS]", |
| | "lstrip": false, |
| | "normalized": false, |
| | "rstrip": false, |
| | "single_word": false |
| | }, |
| | "mask_token": { |
| | "content": "[MASK]", |
| | "lstrip": false, |
| | "normalized": false, |
| | "rstrip": false, |
| | "single_word": false |
| | }, |
| | "pad_token": { |
| | "content": "[PAD]", |
| | "lstrip": false, |
| | "normalized": false, |
| | "rstrip": false, |
| | "single_word": false |
| | }, |
| | "sep_token": { |
| | "content": "[SEP]", |
| | "lstrip": false, |
| | "normalized": false, |
| | "rstrip": false, |
| | "single_word": false |
| | }, |
| | "unk_token": { |
| | "content": "[UNK]", |
| | "lstrip": false, |
| | "normalized": false, |
| | "rstrip": false, |
| | "single_word": false |
| | } |
| | } |
| |
|