File size: 894 Bytes
acced0b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | {
"name": "CRUMB Python BPE \u2014 TOK-1 v7.4.0",
"script_version": "v7.4.0",
"vocab_size": 32768,
"domain": "python_source_code",
"corpus": "stack-dedup-python-parquet-27GB",
"min_frequency": 2,
"trainer": "ByteLevelBPETokenizer via train_from_iterator (Python str)",
"protected_keywords": 38,
"protected_dunders": 21,
"protected_builtins": 73,
"protected_exceptions": 69,
"protected_stdlib": 125,
"protected_ds_tokens": 119,
"total_never_split": 474,
"special_tokens": 20,
"reserved_future_tokens": 4,
"target_model_sizes": [
"124M",
"250M",
"370M",
"500M"
],
"fim_tokens": [
"<fim_prefix>",
"<fim_suffix>",
"<fim_middle>"
],
"notebook_tokens": [
"<cell>",
"</cell>",
"<cell_output>",
"</cell_output>"
],
"comments_in_training": true,
"docstrings_in_training": true,
"training_date": "2026-05-11"
} |