tokenizer_config.json

{
  "files": [
    "data/corpus/base_pretrain_16k.txt"
  ],
  "save_dir": "checkpoints/tokenizer_hf_bpe",
  "vocab_size": 16000,
  "min_frequency": 1,
  "special_tokens": {
    "<|endoftext|>": 0,
    "<|pad|>": 1,
    "<|unk|>": 2,
    "<|system|>": 3,
    "<|user|>": 4,
    "<|assistant|>": 5,
    "<|end_turn|>": 6,
    "<|lang_id|>": 7,
    "<|id|>": 8,
    "<|en|>": 9,
    "<|code|>": 10,
    "<|bos|>": 11,
    "<|eos|>": 12,
    "<|sep|>": 13,
    "<|lang:id|>": 14,
    "<|lang:en|>": 15,
    "<|lang:lampung_o|>": 16,
    "<|lang:mixed|>": 17,
    "<|domain:general|>": 18,
    "<|domain:code|>": 19,
    "<|domain:math|>": 20,
    "<|domain:laravel|>": 21,
    "<|domain:debug|>": 22,
    "<|domain:safety|>": 23,
    "<|domain:translation|>": 24
  },
  "actual_vocab_size": 16000,
  "backend": "hf_bpe"
}