tokenizer_config.json

{
  "model_type": "vortex",
  "tokenizer_class": "VortexTokenizer",
  "auto_map": {
    "AutoTokenizer": ["tokenization_vortex.VortexTokenizer", null]
  },
  "vocab_size": 8192,
  "unk_token": "<unk>",
  "eos_token": "</s>",
  "pad_token": "</s>",
  "model_max_length": 4096,
  "padding_side": "left",
  "chat_template": "{%- if messages and messages[0]['role'] != 'system' %}[SYSTEM]\nYou are a helpful assistant. Answer clearly and say when information is missing.\n</s>\n{%- endif %}{%- for message in messages %}[{{ message['role']|upper }}]\n{{ message['content'] }}\n</s>\n{%- endfor %}{%- if add_generation_prompt %}[ASSISTANT]\n{%- endif %}",
  "add_bos_token": false,
  "add_eos_token": false
}