MyModel-sft / tokenizer_config.json
HappyBulingling's picture
Upload MyModel checkpoint
72f5707 verified
Raw History Blame Contribute Delete
23.6 kB
{ "add_bos_token": false, "add_eos_token": false, "add_prefix_space": false, "added_tokens_decoder": { "0": { "content": "<|eos|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "1": { "content": "<|pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "<|bos|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "3": { "content": "<|unk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "4": { "content": "<|im_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "5": { "content": "<|im_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "6": { "content": "<|system|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "7": { "content": "<|user|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "8": { "content": "<|assistant|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "9": { "content": "<|tool|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "10": { "content": "<think>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "11": { "content": "</think>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "12": { "content": "<tool_call>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "13": { "content": "</tool_call>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "14": { "content": "<tool_response>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "15": { "content": "</tool_response>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16": { "content": "<image_start>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "17": { "content": "<image_pad>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "18": { "content": "<image_end>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "19": { "content": "<audio_start>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "20": { "content": "<audio_pad>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "21": { "content": "<audio_end>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "22": { "content": "<speech_bos>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "23": { "content": "<speech_eos>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "24": { "content": "<speech_pad>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "25": { "content": "<|reserved_1|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "26": { "content": "<|reserved_2|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "27": { "content": "<|reserved_3|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "28": { "content": "<|reserved_4|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "29": { "content": "<|reserved_5|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "30": { "content": "<|reserved_6|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "31": { "content": "<|reserved_7|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "32": { "content": "<|reserved_8|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "33": { "content": "<|reserved_9|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "34": { "content": "<|reserved_10|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "35": { "content": "<|reserved_11|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "36": { "content": "<|reserved_12|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "37": { "content": "<|reserved_13|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "38": { "content": "<|reserved_14|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "39": { "content": "<|reserved_15|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "40": { "content": "<|reserved_16|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "41": { "content": "<|reserved_17|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "42": { "content": "<|reserved_18|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "43": { "content": "<|reserved_19|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "44": { "content": "<|reserved_20|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "45": { "content": "<|reserved_21|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "46": { "content": "<|reserved_22|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "47": { "content": "<|reserved_23|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "48": { "content": "<|reserved_24|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "49": { "content": "<|reserved_25|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50": { "content": "<|reserved_26|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "51": { "content": "<|reserved_27|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "52": { "content": "<|reserved_28|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "53": { "content": "<|reserved_29|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "54": { "content": "<|reserved_30|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "55": { "content": "<|reserved_31|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "56": { "content": "<|reserved_32|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "57": { "content": "<|reserved_33|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "58": { "content": "<|reserved_34|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "59": { "content": "<|reserved_35|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "60": { "content": "<|reserved_36|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "61": { "content": "<|reserved_37|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "62": { "content": "<|reserved_38|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "63": { "content": "<|reserved_39|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "64": { "content": "<|reserved_40|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "65": { "content": "<|reserved_41|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "66": { "content": "<|reserved_42|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "67": { "content": "<|reserved_43|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "68": { "content": "<|reserved_44|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "69": { "content": "<|reserved_45|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "70": { "content": "<|reserved_46|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "71": { "content": "<|reserved_47|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "72": { "content": "<|reserved_48|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "73": { "content": "<|reserved_49|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "74": { "content": "<|reserved_50|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "75": { "content": "<|reserved_51|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "76": { "content": "<|reserved_52|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "77": { "content": "<|reserved_53|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "78": { "content": "<|reserved_54|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "79": { "content": "<|reserved_55|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "80": { "content": "<|reserved_56|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "81": { "content": "<|reserved_57|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "82": { "content": "<|reserved_58|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "83": { "content": "<|reserved_59|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "84": { "content": "<|reserved_60|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "85": { "content": "<|reserved_61|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "86": { "content": "<|reserved_62|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "87": { "content": "<|reserved_63|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "88": { "content": "<|reserved_64|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "89": { "content": "<|reserved_65|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "90": { "content": "<|reserved_66|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "91": { "content": "<|reserved_67|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "92": { "content": "<|reserved_68|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "93": { "content": "<|reserved_69|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "94": { "content": "<|reserved_70|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "95": { "content": "<|reserved_71|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "96": { "content": "<|reserved_72|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "97": { "content": "<|reserved_73|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "98": { "content": "<|reserved_74|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "99": { "content": "<|reserved_75|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100": { "content": "<|reserved_76|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "101": { "content": "<|reserved_77|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "102": { "content": "<|reserved_78|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "103": { "content": "<|reserved_79|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "104": { "content": "<|reserved_80|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "105": { "content": "<|reserved_81|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "106": { "content": "<|reserved_82|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "107": { "content": "<|reserved_83|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "108": { "content": "<|reserved_84|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "109": { "content": "<|reserved_85|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "110": { "content": "<|reserved_86|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "111": { "content": "<|reserved_87|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "112": { "content": "<|reserved_88|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "113": { "content": "<|reserved_89|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "114": { "content": "<|reserved_90|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "115": { "content": "<|reserved_91|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "116": { "content": "<|reserved_92|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "117": { "content": "<|reserved_93|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "118": { "content": "<|reserved_94|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "119": { "content": "<|reserved_95|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "120": { "content": "<|reserved_96|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "121": { "content": "<|reserved_97|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "122": { "content": "<|reserved_98|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "123": { "content": "<|reserved_99|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "124": { "content": "<|reserved_100|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "125": { "content": "<|reserved_101|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "126": { "content": "<|reserved_102|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "127": { "content": "<|reserved_103|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "additional_special_tokens": [ "<|im_start|>", "<|im_end|>", "<|system|>", "<|user|>", "<|assistant|>", "<|tool|>", "<think>", "</think>", "<tool_call>", "</tool_call>", "<tool_response>", "</tool_response>", "<image_start>", "<image_pad>", "<image_end>", "<audio_start>", "<audio_pad>", "<audio_end>", "<speech_bos>", "<speech_eos>", "<speech_pad>" ], "bos_token": "<|bos|>", "clean_up_tokenization_spaces": false, "eos_token": "<|eos|>", "extra_special_tokens": {}, "legacy": true, "model_max_length": 4096, "pad_token": "<|pad|>", "spaces_between_special_tokens": false, "tokenizer_class": "PreTrainedTokenizerFast", "unk_token": "<|unk|>" }
{
"add_bos_token": false,
"add_eos_token": false,
"add_prefix_space": false,
"added_tokens_decoder": {
"0": {
"content": "<|eos|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<|pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "<|bos|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "<|unk|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"4": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"5": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"6": {
"content": "<|system|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"7": {
"content": "<|user|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"8": {
"content": "<|assistant|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"9": {
"content": "<|tool|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"10": {
"content": "<think>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"11": {
"content": "</think>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true