| { |
| "added_tokens_decoder": { |
| "25460": { |
| "content": "<|startoftext|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25461": { |
| "content": "<|endoftext|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25462": { |
| "content": "<|reserved_200000|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25463": { |
| "content": "<|reserved_200001|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25464": { |
| "content": "<|return|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25465": { |
| "content": "<|constrain|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25466": { |
| "content": "<|reserved_200004|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25467": { |
| "content": "<|channel|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25468": { |
| "content": "<|start|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25469": { |
| "content": "<|end|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25470": { |
| "content": "<|message|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25471": { |
| "content": "<|reserved_200009|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25472": { |
| "content": "<|reserved_200010|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25473": { |
| "content": "<|reserved_200011|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25474": { |
| "content": "<|call|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25475": { |
| "content": "<|reserved_200013|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25476": { |
| "content": "<|reserved_200014|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25477": { |
| "content": "<|reserved_200015|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25478": { |
| "content": "<|reserved_200016|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25479": { |
| "content": "<|reserved_200017|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25480": { |
| "content": "<|endofprompt|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25481": { |
| "content": "[MASK]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "25482": { |
| "content": "<|reserved_200020|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25483": { |
| "content": "<|reserved_200021|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25484": { |
| "content": "<|reserved_200022|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25485": { |
| "content": "<|reserved_200023|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25486": { |
| "content": "<|reserved_200024|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25487": { |
| "content": "<|reserved_200025|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25488": { |
| "content": "<|reserved_200026|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25489": { |
| "content": "<|reserved_200027|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25490": { |
| "content": "<|reserved_200028|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25491": { |
| "content": "<|reserved_200029|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25492": { |
| "content": "<|reserved_200030|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25493": { |
| "content": "<|reserved_200031|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25494": { |
| "content": "<|reserved_200032|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25495": { |
| "content": "<|reserved_200033|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25496": { |
| "content": "<|reserved_200034|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25497": { |
| "content": "<|reserved_200035|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25498": { |
| "content": "<|reserved_200036|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25499": { |
| "content": "<|reserved_200037|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25500": { |
| "content": "<|reserved_200038|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25501": { |
| "content": "<|reserved_200039|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25502": { |
| "content": "<|reserved_200040|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25503": { |
| "content": "<|reserved_200041|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25504": { |
| "content": "<|reserved_200042|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25505": { |
| "content": "<|reserved_200043|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25506": { |
| "content": "<|reserved_200044|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25507": { |
| "content": "<|reserved_200045|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25508": { |
| "content": "<|reserved_200046|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25509": { |
| "content": "<|reserved_200047|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25510": { |
| "content": "<|reserved_200048|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25511": { |
| "content": "<|reserved_200049|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25512": { |
| "content": "<|reserved_200050|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25513": { |
| "content": "<|reserved_200051|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25514": { |
| "content": "<|reserved_200052|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25515": { |
| "content": "<|reserved_200053|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25516": { |
| "content": "<|reserved_200054|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25517": { |
| "content": "<|reserved_200055|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25518": { |
| "content": "<|reserved_200056|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25519": { |
| "content": "<|reserved_200057|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25520": { |
| "content": "<|reserved_200058|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25521": { |
| "content": "<|reserved_200059|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25522": { |
| "content": "<|reserved_200060|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25523": { |
| "content": "<|reserved_200061|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25524": { |
| "content": "<|reserved_200062|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| }, |
| "25525": { |
| "content": "<|reserved_200063|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": true, |
| "special": false |
| } |
| }, |
| "backend": "tokenizers", |
| "bos_token": "<|startoftext|>", |
| "clean_up_tokenization_spaces": false, |
| "cls_token": "<|startoftext|>", |
| "eos_token": "<|return|>", |
| "extra_special_tokens": {}, |
| "is_local": true, |
| "mask_token": "[MASK]", |
| "max_length": 32768, |
| "model_input_names": [ |
| "input_ids", |
| "attention_mask" |
| ], |
| "model_max_length": 32768, |
| "pad_to_multiple_of": null, |
| "pad_token": "<|endoftext|>", |
| "pad_token_type_id": 0, |
| "padding_side": "right", |
| "sep_token": "<|return|>", |
| "stride": 0, |
| "tokenizer_class": "PreTrainedTokenizerFast", |
| "truncation_side": "right", |
| "truncation_strategy": "longest_first" |
| } |
