| { | |
| "vocab_size": 32768, | |
| "requested_vocab_size": 32768, | |
| "pad_token": "<pad>", | |
| "eos_token": "<|endoftext|>", | |
| "pad_id": 0, | |
| "eos_id": 1, | |
| "sample": "/home/gmachine/data/pretrain_packed/tokenizer_sample.jsonl", | |
| "train_seconds": 154.5, | |
| "backend": "huggingface-tokenizers-bytelevel-bpe" | |
| } |