Upload tokenizer
b786b21 verified | | { |
| | "bos_token": { |
| | "content": "<s>", |
| | "lstrip": false, |
| | "normalized": false, |
| | "rstrip": false, |
| | "single_word": false |
| | }, |
| | "eos_token": { |
| | "content": "<|im_end|>", |
| | "lstrip": false, |
| | "normalized": false, |
| | "rstrip": false, |
| | "single_word": false |
| | }, |
| | "pad_token": { |
| | "content": "</s>", |
| | "lstrip": false, |
| | "normalized": false, |
| | "rstrip": false, |
| | "single_word": false |
| | }, |
| | "unk_token": { |
| | "content": "<unk>", |
| | "lstrip": false, |
| | "normalized": false, |
| | "rstrip": false, |
| | "single_word": false |
| | } |
| | } |
| |
|