tokenizer_config.json

{
  "backend": "tokenizers",
  "bos_token": "<s>",
  "clean_up_tokenization_spaces": true,
  "eos_token": "</s>",
  "extra_special_tokens": [
    "<user>",
    "<assistant>",
    "<system>",
    "<sudan>",
    "<url>",
    "<mention>",
    "<hashtag>",
    "<elongation>",
    "<emoji_purple_heart>",
    "<emoji_sleep>",
    "<emoji_money>",
    "<emoji_confetti>",
    "<emoji_trophy>",
    "<emoji_heart>",
    "<emoji_broken_heart>",
    "<emoji_angry>",
    "<emoji_soccer>",
    "<emoji_camera>",
    "<emoji_arrow_right>",
    "<emoji_hearts>",
    "<emoji_tea>",
    "<emoji_green_circle>",
    "<emoji_ok>",
    "<emoji_sparkles>",
    "<emoji_party>",
    "<emoji_star>",
    "<emoji_green_heart>",
    "<emoji_notes>",
    "<emoji_joy>",
    "<emoji_sad>",
    "<emoji_wow>",
    "<emoji_rofl>",
    "<emoji_sweat>",
    "<emoji_check>",
    "<emoji_100>",
    "<emoji_love>",
    "<emoji_mask>",
    "<emoji_thumbsdown>",
    "<emoji_bell>",
    "<emoji_blue_heart>",
    "<emoji_rage>",
    "<emoji_grin>",
    "<emoji_black_heart>",
    "<emoji_sun>",
    "<emoji_white_heart>",
    "<emoji_cross>",
    "<emoji_rainbow>",
    "<emoji_red_circle>",
    "<emoji_arrow_left>",
    "<emoji_shock>",
    "<emoji_music>",
    "<emoji_phone>",
    "<emoji_star2>",
    "<emoji_neutral>",
    "<emoji_coffee>",
    "<emoji_fire>",
    "<emoji_clapper>",
    "<emoji_moon>",
    "<emoji_clap>",
    "<emoji_rose>",
    "<emoji_gift_heart>",
    "<emoji_mic>",
    "<emoji_pray>",
    "<emoji_yellow_heart>",
    "<emoji_computer>",
    "<emoji_think>",
    "<emoji_cry>",
    "<emoji_thumbsup>",
    "<emoji_smile>"
  ],
  "mask_token": "<mask>",
  "model_max_length": 1000000000000000019884624838656,
  "pad_token": "<pad>",
  "tokenizer_class": "TokenizersBackend",
  "unk_token": "<unk>"
}