Hugging Face's logo

crumb-ai
/
crumb-python-bpe-tokenizer

Model card Files
xet
 Community
crumb-python-bpe-tokenizer
File size: 1,606 Bytes
acced0b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
{
  "backend": "tokenizers",
  "bos_token": "<s>",
  "eos_token": "</s>",
  "extra_special_tokens": [
    "<func>",
    "</func>",
    "<class>",
    "</class>",
    "<module>",
    "</module>",
    "<cell>",
    "</cell>",
    "<cell_output>",
    "</cell_output>",
    "<traceback>",
    "</traceback>",
    "<fim_prefix>",
    "<fim_suffix>",
    "<fim_middle>",
    "<comment>",
    "<docstring>",
    "</docstring>",
    "<type_hint>"
  ],
  "mask_token": "<mask>",
  "model_max_length": 1000000000000000019884624838656,
  "pad_token": "<pad>",
  "tokenizer_class": "TokenizersBackend",
  "unk_token": "<unk>",
  "crumb_metadata": {
    "name": "CRUMB Python BPE \u2014 TOK-1 v7.4.0",
    "script_version": "v7.4.0",
    "vocab_size": 32768,
    "domain": "python_source_code",
    "corpus": "stack-dedup-python-parquet-27GB",
    "min_frequency": 2,
    "trainer": "ByteLevelBPETokenizer via train_from_iterator (Python str)",
    "protected_keywords": 38,
    "protected_dunders": 21,
    "protected_builtins": 73,
    "protected_exceptions": 69,
    "protected_stdlib": 125,
    "protected_ds_tokens": 119,
    "total_never_split": 474,
    "special_tokens": 20,
    "reserved_future_tokens": 4,
    "target_model_sizes": [
      "124M",
      "250M",
      "370M",
      "500M"
    ],
    "fim_tokens": [
      "<fim_prefix>",
      "<fim_suffix>",
      "<fim_middle>"
    ],
    "notebook_tokens": [
      "<cell>",
      "</cell>",
      "<cell_output>",
      "</cell_output>"
    ],
    "comments_in_training": true,
    "docstrings_in_training": true,
    "training_date": "2026-05-11"
  }
}