{
  "vocab_size": 32768,
  "requested_vocab_size": 32768,
  "pad_token": "<pad>",
  "eos_token": "<|endoftext|>",
  "pad_id": 0,
  "eos_id": 1,
  "sample": "/home/gmachine/data/pretrain_packed/tokenizer_sample.jsonl",
  "train_seconds": 154.5,
  "backend": "huggingface-tokenizers-bytelevel-bpe"
}