tokenizer_config.json

{
  "vocab_size": 32000,
  "model_type": "bpe",
  "special_tokens": [
    "<|pad|>",
    "<|unk|>",
    "<|bos|>",
    "<|eos|>",
    "<|system|>",
    "<|user|>",
    "<|assistant|>",
    "<|thinking|>",
    "<|intent|>",
    "<|quality|>",
    "<|context|>",
    "<|end_of_thinking|>"
  ],
  "training_datasets": [
    "bitext/Bitext-customer-support-llm-chatbot-training-dataset",
    "databricks/databricks-dolly-15k",
    "tatsu-lab/alpaca",
    "clinc/clinc_oos",
    "amazon_polarity",
    "ag_news",
    "imdb",
    "yelp_review_full",
    "dgbillings/stackoverflow-questions",
    "Open-Orca/OpenOrca"
  ]
}