tokenizer_meta.json

{
  "vocab_size": 32000,
  "type": "BPE",
  "individual_digits": true,
  "pre_tokenizer": "Digits(individual=True) + ByteLevel",
  "normalizer": "NFKC + whitespace",
  "purpose": "English + Math reasoning for SonaMath",
  "special_tokens": [
    "<pad>",
    "<bos>",
    "<eos>",
    "<unk>",
    "<think>",
    "</think>",
    "<step>",
    "</step>",
    "<answer>",
    "</answer>",
    "<equation>",
    "</equation>",
    "<op>",
    "</op>",
    "<lang>",
    "</lang>"
  ]
}