| { |
| "add_prefix_space": false, |
| "backend": "tokenizers", |
| "bos_token": "<|endoftext|>", |
| "clean_up_tokenization_spaces": false, |
| "eos_token": "<|endoftext|>", |
| "is_local": false, |
| "local_files_only": false, |
| "model_max_length": 65536, |
| "pad_token": "<|pad|>", |
| "padding_side": "left", |
| "tokenizer_class": "TokenizersBackend", |
| "unk_token": "�", |
| "added_tokens_decoder": { |
| "5809": { |
| "content": "�", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| "100256": { |
| "content": "<|extra_id_0|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": false |
| }, |
| "100257": { |
| "content": "<|endoftext|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| "100258": { |
| "content": "<|fim_prefix|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| "100259": { |
| "content": "<|fim_middle|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| "100260": { |
| "content": "<|fim_suffix|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| "100261": { |
| "content": "|||PHONE_NUMBER|||", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": false |
| }, |
| "100262": { |
| "content": "|||EMAIL_ADDRESS|||", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": false |
| }, |
| "100263": { |
| "content": "|||IP_ADDRESS|||", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": false |
| }, |
| "100264": { |
| "content": "<|im_start|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| "100265": { |
| "content": "<|im_end|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| "100266": { |
| "content": "<functions>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": false |
| }, |
| "100267": { |
| "content": "</functions>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": false |
| }, |
| "100268": { |
| "content": "<function_calls>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": false |
| }, |
| "100269": { |
| "content": "</function_calls>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": false |
| }, |
| "100270": { |
| "content": "<|extra_id_1|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": false |
| }, |
| "100271": { |
| "content": "<|extra_id_2|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": false |
| }, |
| "100272": { |
| "content": "<|extra_id_3|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": false |
| }, |
| "100273": { |
| "content": "<|extra_id_4|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": false |
| }, |
| "100274": { |
| "content": "<|extra_id_5|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": false |
| }, |
| "100275": { |
| "content": "<|extra_id_6|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": false |
| }, |
| "100276": { |
| "content": "<|endofprompt|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| "100277": { |
| "content": "<|pad|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| } |
| } |
| } |
|
|
