compiler/tokenizer_config.json

{
  "add_prefix_space": false,
  "backend": "tokenizers",
  "bos_token": null,
  "clean_up_tokenization_spaces": false,
  "eos_token": "<|im_end|>",
  "errors": "replace",
  "extra_special_tokens": {
    "<prefix_1>": "<prefix_1>",
    "<prefix_2>": "<prefix_2>",
    "<prefix_3>": "<prefix_3>",
    "<prefix_4>": "<prefix_4>",
    "<prefix_5>": "<prefix_5>",
    "<prefix_6>": "<prefix_6>",
    "<prefix_7>": "<prefix_7>",
    "<prefix_8>": "<prefix_8>",
    "<prefix_9>": "<prefix_9>",
    "<prefix_10>": "<prefix_10>",
    "<prefix_11>": "<prefix_11>",
    "<prefix_12>": "<prefix_12>",
    "<prefix_13>": "<prefix_13>",
    "<prefix_14>": "<prefix_14>",
    "<prefix_15>": "<prefix_15>",
    "<prefix_16>": "<prefix_16>",
    "<prefix_17>": "<prefix_17>",
    "<prefix_18>": "<prefix_18>",
    "<prefix_19>": "<prefix_19>",
    "<prefix_20>": "<prefix_20>",
    "<prefix_21>": "<prefix_21>",
    "<prefix_22>": "<prefix_22>",
    "<prefix_23>": "<prefix_23>",
    "<prefix_24>": "<prefix_24>",
    "<prefix_25>": "<prefix_25>",
    "<prefix_26>": "<prefix_26>",
    "<prefix_27>": "<prefix_27>",
    "<prefix_28>": "<prefix_28>",
    "<prefix_29>": "<prefix_29>",
    "<prefix_30>": "<prefix_30>",
    "<prefix_31>": "<prefix_31>",
    "<prefix_32>": "<prefix_32>",
    "<prefix_33>": "<prefix_33>",
    "<prefix_34>": "<prefix_34>",
    "<prefix_35>": "<prefix_35>",
    "<prefix_36>": "<prefix_36>",
    "<prefix_37>": "<prefix_37>",
    "<prefix_38>": "<prefix_38>",
    "<prefix_39>": "<prefix_39>",
    "<prefix_40>": "<prefix_40>",
    "<prefix_41>": "<prefix_41>",
    "<prefix_42>": "<prefix_42>",
    "<prefix_43>": "<prefix_43>",
    "<prefix_44>": "<prefix_44>",
    "<prefix_45>": "<prefix_45>",
    "<prefix_46>": "<prefix_46>",
    "<prefix_47>": "<prefix_47>",
    "<prefix_48>": "<prefix_48>",
    "<prefix_49>": "<prefix_49>",
    "<prefix_50>": "<prefix_50>",
    "<prefix_51>": "<prefix_51>",
    "<prefix_52>": "<prefix_52>",
    "<prefix_53>": "<prefix_53>",
    "<prefix_54>": "<prefix_54>",
    "<prefix_55>": "<prefix_55>",
    "<prefix_56>": "<prefix_56>",
    "<prefix_57>": "<prefix_57>",
    "<prefix_58>": "<prefix_58>",
    "<prefix_59>": "<prefix_59>",
    "<prefix_60>": "<prefix_60>",
    "<prefix_61>": "<prefix_61>",
    "<prefix_62>": "<prefix_62>",
    "<prefix_63>": "<prefix_63>",
    "<prefix_64>": "<prefix_64>"
  },
  "is_local": true,
  "model_max_length": 1010000,
  "pad_token": "<|endoftext|>",
  "split_special_tokens": false,
  "tokenizer_class": "Qwen2Tokenizer",
  "unk_token": null
}