lm_weights_scratch / tokenizer_config.json
zuhri025's picture
Move checkpoint-24000 files to root
16a4b79 verified
{
"backend": "tokenizers",
"bos_token": "<|bos|>",
"eos_token": "<|eos|>",
"extra_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|system|>",
"<|user|>",
"<|assistant|>",
"<|speech|>",
"<|frame_start|>",
"<|frame_end|>",
"<|reserved_1|>",
"<|reserved_2|>",
"<|reserved_3|>",
"<|reserved_4|>",
"<|reserved_5|>",
"<|reserved_6|>",
"<|reserved_7|>",
"<|reserved_8|>",
"<|reserved_9|>",
"<|reserved_10|>",
"<|reserved_11|>",
"<|reserved_12|>",
"<|reserved_13|>",
"<|reserved_14|>",
"<|reserved_15|>",
"<|reserved_16|>",
"<|reserved_17|>",
"<|reserved_18|>",
"<|reserved_19|>",
"<|reserved_20|>",
"<|reserved_21|>",
"<|reserved_22|>",
"<|reserved_23|>",
"<|reserved_24|>",
"<|reserved_25|>",
"<|reserved_26|>",
"<|reserved_27|>",
"<|reserved_28|>",
"<|reserved_29|>",
"<|reserved_30|>",
"<|reserved_31|>",
"<|reserved_32|>",
"<|reserved_33|>",
"<|reserved_34|>",
"<|reserved_35|>",
"<|reserved_36|>",
"<|reserved_37|>",
"<|reserved_38|>",
"<|reserved_39|>",
"<|reserved_40|>",
"<|reserved_41|>",
"<|reserved_42|>",
"<|reserved_43|>",
"<|reserved_44|>",
"<|reserved_45|>",
"<|reserved_46|>",
"<|reserved_47|>",
"<|reserved_48|>",
"<|reserved_49|>",
"<|reserved_50|>",
"<|reserved_51|>",
"<|reserved_52|>",
"<|reserved_53|>",
"<|reserved_54|>",
"<|reserved_55|>",
"<|reserved_56|>",
"<|reserved_57|>",
"<|reserved_58|>",
"<|reserved_59|>",
"<|reserved_60|>",
"<|reserved_61|>",
"<|reserved_62|>",
"<|reserved_63|>",
"<|reserved_64|>",
"<|reserved_65|>",
"<|reserved_66|>",
"<|reserved_67|>",
"<|reserved_68|>",
"<|reserved_69|>",
"<|reserved_70|>",
"<|reserved_71|>",
"<|reserved_72|>",
"<|reserved_73|>",
"<|reserved_74|>",
"<|reserved_75|>",
"<|reserved_76|>",
"<|reserved_77|>",
"<|reserved_78|>",
"<|reserved_79|>",
"<|reserved_80|>"
],
"is_local": true,
"model_max_length": 1000000000000000019884624838656,
"pad_token": "<|pad|>",
"tokenizer_class": "TokenizersBackend",
"unk_token": "<|unk|>"
}