Hugging Face's logo

Xerv-AI
/
CRAYON-tokenizer

Text Generation
English
generic
tokenizers
hardware-accelerated
avx2
cuda
rocm
double-array-trie
crayon
Model card Files
xet
 Community
CRAYON-tokenizer
File size: 6,427 Bytes
708f4a3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
[
  {
    "impl": "crayon:cpu:lite",
    "case": "english",
    "status": "OK",
    "load_time_ms": 293.8474006950855,
    "tokens_produced": 144001,
    "bytes_processed": 740000,
    "avg_time_ms": 188.5325202718377,
    "tokens_per_sec": 763799.2628134954,
    "mb_per_sec": 3.743221557337038,
    "notes": ""
  },
  {
    "impl": "crayon:cpu:lite",
    "case": "code",
    "status": "OK",
    "load_time_ms": 53.46729978919029,
    "tokens_produced": 556000,
    "bytes_processed": 1048000,
    "avg_time_ms": 137.21070028841496,
    "tokens_per_sec": 4052162.104203942,
    "mb_per_sec": 7.284057886833307,
    "notes": ""
  },
  {
    "impl": "crayon:cpu:lite",
    "case": "unicode",
    "status": "OK",
    "load_time_ms": 34.77140050381422,
    "tokens_produced": 474000,
    "bytes_processed": 660000,
    "avg_time_ms": 85.59199962764978,
    "tokens_per_sec": 5537900.762478252,
    "mb_per_sec": 7.353783666304187,
    "notes": ""
  },
  {
    "impl": "crayon:cpu:lite",
    "case": "mixed",
    "status": "OK",
    "load_time_ms": 44.43740006536245,
    "tokens_produced": 640000,
    "bytes_processed": 1397500,
    "avg_time_ms": 214.90477975457907,
    "tokens_per_sec": 2978063.1251239693,
    "mb_per_sec": 6.201629664541404,
    "notes": ""
  },
  {
    "impl": "crayon:cpu:standard",
    "case": "english",
    "status": "OK",
    "load_time_ms": 256.83970004320145,
    "tokens_produced": 136001,
    "bytes_processed": 740000,
    "avg_time_ms": 45.62829993665218,
    "tokens_per_sec": 2980628.25458797,
    "mb_per_sec": 15.466694904706209,
    "notes": ""
  },
  {
    "impl": "crayon:cpu:standard",
    "case": "code",
    "status": "OK",
    "load_time_ms": 373.47840052098036,
    "tokens_produced": 312000,
    "bytes_processed": 1048000,
    "avg_time_ms": 105.5902199819684,
    "tokens_per_sec": 2954819.111592722,
    "mb_per_sec": 9.465371733901357,
    "notes": ""
  },
  {
    "impl": "crayon:cpu:standard",
    "case": "unicode",
    "status": "OK",
    "load_time_ms": 289.2025001347065,
    "tokens_produced": 216001,
    "bytes_processed": 660000,
    "avg_time_ms": 32.941219955682755,
    "tokens_per_sec": 6557164.558282767,
    "mb_per_sec": 19.107520901621665,
    "notes": ""
  },
  {
    "impl": "crayon:cpu:standard",
    "case": "mixed",
    "status": "OK",
    "load_time_ms": 161.32330056279898,
    "tokens_produced": 372501,
    "bytes_processed": 1397500,
    "avg_time_ms": 204.93176002055407,
    "tokens_per_sec": 1817683.115406998,
    "mb_per_sec": 6.503432445239636,
    "notes": ""
  },
  {
    "impl": "tiktoken:p50k_base",
    "case": "english",
    "status": "OK",
    "load_time_ms": 971.7505006119609,
    "tokens_produced": 144001,
    "bytes_processed": 740000,
    "avg_time_ms": 961.8989199399948,
    "tokens_per_sec": 149704.91910832282,
    "mb_per_sec": 0.7336727170716121,
    "notes": ""
  },
  {
    "impl": "tiktoken:p50k_base",
    "case": "code",
    "status": "OK",
    "load_time_ms": 0.019999220967292786,
    "tokens_produced": 420000,
    "bytes_processed": 1048000,
    "avg_time_ms": 2896.1911803111434,
    "tokens_per_sec": 145018.0508991394,
    "mb_per_sec": 0.34509140501090024,
    "notes": ""
  },
  {
    "impl": "tiktoken:p50k_base",
    "case": "unicode",
    "status": "OK",
    "load_time_ms": 0.013899989426136017,
    "tokens_produced": 378000,
    "bytes_processed": 660000,
    "avg_time_ms": 1686.671899817884,
    "tokens_per_sec": 224109.97659996233,
    "mb_per_sec": 0.3731757485828075,
    "notes": ""
  },
  {
    "impl": "tiktoken:p50k_base",
    "case": "mixed",
    "status": "OK",
    "load_time_ms": 0.00980030745267868,
    "tokens_produced": 517500,
    "bytes_processed": 1397500,
    "avg_time_ms": 5820.50839997828,
    "tokens_per_sec": 88909.75915470393,
    "mb_per_sec": 0.22897653702942988,
    "notes": ""
  },
  {
    "impl": "tiktoken:cl100k_base",
    "case": "english",
    "status": "OK",
    "load_time_ms": 1833.9411998167634,
    "tokens_produced": 140001,
    "bytes_processed": 740000,
    "avg_time_ms": 1069.1874599084258,
    "tokens_per_sec": 130941.49085137124,
    "mb_per_sec": 0.6600516940229253,
    "notes": ""
  },
  {
    "impl": "tiktoken:cl100k_base",
    "case": "code",
    "status": "OK",
    "load_time_ms": 0.006299465894699097,
    "tokens_produced": 308000,
    "bytes_processed": 1048000,
    "avg_time_ms": 2401.132099889219,
    "tokens_per_sec": 128272.82597830008,
    "mb_per_sec": 0.4162414402938771,
    "notes": ""
  },
  {
    "impl": "tiktoken:cl100k_base",
    "case": "unicode",
    "status": "OK",
    "load_time_ms": 0.010799616575241089,
    "tokens_produced": 306000,
    "bytes_processed": 660000,
    "avg_time_ms": 1080.0928801298141,
    "tokens_per_sec": 283308.96872796945,
    "mb_per_sec": 0.5827508544936206,
    "notes": ""
  },
  {
    "impl": "tiktoken:cl100k_base",
    "case": "mixed",
    "status": "OK",
    "load_time_ms": 0.0112997367978096,
    "tokens_produced": 410000,
    "bytes_processed": 1397500,
    "avg_time_ms": 2310.1005997508764,
    "tokens_per_sec": 177481.4482296636,
    "mb_per_sec": 0.576927194132351,
    "notes": ""
  },
  {
    "impl": "tiktoken:o200k_base",
    "case": "english",
    "status": "OK",
    "load_time_ms": 2667.899999767542,
    "tokens_produced": 140001,
    "bytes_processed": 740000,
    "avg_time_ms": 881.4683999866247,
    "tokens_per_sec": 158827.0209143338,
    "mb_per_sec": 0.8006174630325188,
    "notes": ""
  },
  {
    "impl": "tiktoken:o200k_base",
    "case": "code",
    "status": "OK",
    "load_time_ms": 0.0061998143792152405,
    "tokens_produced": 312000,
    "bytes_processed": 1048000,
    "avg_time_ms": 1776.3976402580738,
    "tokens_per_sec": 175636.35130402044,
    "mb_per_sec": 0.5626277928676771,
    "notes": ""
  },
  {
    "impl": "tiktoken:o200k_base",
    "case": "unicode",
    "status": "OK",
    "load_time_ms": 0.00509992241859436,
    "tokens_produced": 210000,
    "bytes_processed": 660000,
    "avg_time_ms": 647.867819853127,
    "tokens_per_sec": 324140.192744266,
    "mb_per_sec": 0.971533126881988,
    "notes": ""
  },
  {
    "impl": "tiktoken:o200k_base",
    "case": "mixed",
    "status": "OK",
    "load_time_ms": 0.006099231541156769,
    "tokens_produced": 372500,
    "bytes_processed": 1397500,
    "avg_time_ms": 1556.8332800641656,
    "tokens_per_sec": 239267.75253972426,
    "mb_per_sec": 0.8560710220190078,
    "notes": ""
  }
]