File size: 1,278 Bytes
11e0d89 02eef51 11e0d89 02eef51 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | {
"add_prefix_space": false,
"backend": "tokenizers",
"bos_token": null,
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"errors": "replace",
"extra_special_tokens": {
"<|mindi_start|>": "<|mindi_start|>",
"<|mindi_end|>": "<|mindi_end|>",
"<|code_start|>": "<|code_start|>",
"<|code_end|>": "<|code_end|>",
"<|vision_start|>": "<|vision_start|>",
"<|vision_end|>": "<|vision_end|>",
"<|critique_start|>": "<|critique_start|>",
"<|critique_end|>": "<|critique_end|>",
"<|suggest_start|>": "<|suggest_start|>",
"<|suggest_end|>": "<|suggest_end|>",
"<|think_start|>": "<|think_start|>",
"<|think_end|>": "<|think_end|>",
"<|file_start|>": "<|file_start|>",
"<|file_end|>": "<|file_end|>",
"<|search_start|>": "<|search_start|>",
"<|search_end|>": "<|search_end|>",
"<|sandbox_start|>": "<|sandbox_start|>",
"<|sandbox_end|>": "<|sandbox_end|>",
"<|error_start|>": "<|error_start|>",
"<|error_end|>": "<|error_end|>",
"<|fix_start|>": "<|fix_start|>",
"<|fix_end|>": "<|fix_end|>"
},
"is_local": true,
"model_max_length": 32768,
"pad_token": "<|endoftext|>",
"split_special_tokens": false,
"tokenizer_class": "Qwen2Tokenizer",
"unk_token": null
} |