tokenizer_config.json

{
  "backend": "tokenizers",
  "bos_token": "<s>",
  "eos_token": "</s>",
  "extra_special_tokens": [
    "<func>",
    "</func>",
    "<class>",
    "</class>",
    "<module>",
    "</module>",
    "<cell>",
    "</cell>",
    "<cell_output>",
    "</cell_output>",
    "<traceback>",
    "</traceback>",
    "<fim_prefix>",
    "<fim_suffix>",
    "<fim_middle>",
    "<comment>",
    "<docstring>",
    "</docstring>",
    "<type_hint>"
  ],
  "mask_token": "<mask>",
  "model_max_length": 1000000000000000019884624838656,
  "pad_token": "<pad>",
  "tokenizer_class": "TokenizersBackend",
  "unk_token": "<unk>",
  "crumb_metadata": {
    "name": "CRUMB Python BPE \u2014 TOK-1 v7.4.0",
    "script_version": "v7.4.0",
    "vocab_size": 32768,
    "domain": "python_source_code",
    "corpus": "stack-dedup-python-parquet-27GB",
    "min_frequency": 2,
    "trainer": "ByteLevelBPETokenizer via train_from_iterator (Python str)",
    "protected_keywords": 38,
    "protected_dunders": 21,
    "protected_builtins": 73,
    "protected_exceptions": 69,
    "protected_stdlib": 125,
    "protected_ds_tokens": 119,
    "total_never_split": 474,
    "special_tokens": 20,
    "reserved_future_tokens": 4,
    "target_model_sizes": [
      "124M",
      "250M",
      "370M",
      "500M"
    ],
    "fim_tokens": [
      "<fim_prefix>",
      "<fim_suffix>",
      "<fim_middle>"
    ],
    "notebook_tokens": [
      "<cell>",
      "</cell>",
      "<cell_output>",
      "</cell_output>"
    ],
    "comments_in_training": true,
    "docstrings_in_training": true,
    "training_date": "2026-05-11"
  }
}