tokenizer/config.json

{
  "vocab_size": 10000,
  "model_type": "BPE",
  "dataset": "roneneldan/TinyStories",
  "min_frequency": 2,
  "training_samples": 100000,
  "special_tokens": {
    "pad_token": "<|padding|>",
    "eos_token": "<|endoftext|>",
    "unk_token": "<unk>"
  }
}