tokenizer_meta.json

{
  "algorithm": "ByteLevel BPE",
  "requested_vocab_size": 48000,
  "actual_vocab_size": 48000,
  "special_tokens": {
    "<pad>": 0,
    "<s>": 1,
    "</s>": 2,
    "<unk>": 3,
    "<ko>": 4,
    "<en>": 5
  }
}