| { |
| "name": "CRUMB Python BPE \u2014 TOK-1 v7.4.0", |
| "script_version": "v7.4.0", |
| "vocab_size": 32768, |
| "domain": "python_source_code", |
| "corpus": "stack-dedup-python-parquet-27GB", |
| "min_frequency": 2, |
| "trainer": "ByteLevelBPETokenizer via train_from_iterator (Python str)", |
| "protected_keywords": 38, |
| "protected_dunders": 21, |
| "protected_builtins": 73, |
| "protected_exceptions": 69, |
| "protected_stdlib": 125, |
| "protected_ds_tokens": 119, |
| "total_never_split": 474, |
| "special_tokens": 20, |
| "reserved_future_tokens": 4, |
| "target_model_sizes": [ |
| "124M", |
| "250M", |
| "370M", |
| "500M" |
| ], |
| "fim_tokens": [ |
| "<fim_prefix>", |
| "<fim_suffix>", |
| "<fim_middle>" |
| ], |
| "notebook_tokens": [ |
| "<cell>", |
| "</cell>", |
| "<cell_output>", |
| "</cell_output>" |
| ], |
| "comments_in_training": true, |
| "docstrings_in_training": true, |
| "training_date": "2026-05-11" |
| } |
