Hugging Face's logo

ItsMaxNorm
/
bpess

chess,
tiktoken,
tokenizer
Model card Files
xet
Community
1
ItsMaxNorm commited on
Commit
643e9f6
·
verified ·
1 Parent(s): 1c69aaf

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. config.json +1 -0
  2. vocab.json +262 -0
config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"pattern": "'(?i:[sdmt]|ll|ve|re)|[^\\r\\n\\p{L}\\p{N}]?+\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]++[\\r\\n]*|\\s*[\\r\\n]|\\s+(?!\\S)|\\s+", "vocab_size": 388}
vocab.json ADDED
@@ -0,0 +1,262 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "\u0000": 0,
3
+ "\u0001": 1,
4
+ "\u0002": 2,
5
+ "\u0003": 3,
6
+ "\u0004": 4,
7
+ "\u0005": 5,
8
+ "\u0006": 6,
9
+ "\u0007": 7,
10
+ "\b": 8,
11
+ "\t": 9,
12
+ "\n": 10,
13
+ "\u000b": 11,
14
+ "\f": 12,
15
+ "\r": 13,
16
+ "\u000e": 14,
17
+ "\u000f": 15,
18
+ "\u0010": 16,
19
+ "\u0011": 17,
20
+ "\u0012": 18,
21
+ "\u0013": 19,
22
+ "\u0014": 20,
23
+ "\u0015": 21,
24
+ "\u0016": 22,
25
+ "\u0017": 23,
26
+ "\u0018": 24,
27
+ "\u0019": 25,
28
+ "\u001a": 26,
29
+ "\u001b": 27,
30
+ "\u001c": 28,
31
+ "\u001d": 29,
32
+ "\u001e": 30,
33
+ "\u001f": 31,
34
+ " ": 32,
35
+ "!": 33,
36
+ "\"": 34,
37
+ "#": 35,
38
+ "$": 36,
39
+ "%": 37,
40
+ "&": 38,
41
+ "'": 39,
42
+ "(": 40,
43
+ ")": 41,
44
+ "*": 42,
45
+ "+": 43,
46
+ ",": 44,
47
+ "-": 45,
48
+ ".": 46,
49
+ "/": 47,
50
+ "0": 48,
51
+ "1": 49,
52
+ "2": 50,
53
+ "3": 51,
54
+ "4": 52,
55
+ "5": 53,
56
+ "6": 54,
57
+ "7": 55,
58
+ "8": 56,
59
+ "9": 57,
60
+ ":": 58,
61
+ ";": 59,
62
+ "<": 60,
63
+ "=": 61,
64
+ ">": 62,
65
+ "?": 63,
66
+ "@": 64,
67
+ "A": 65,
68
+ "B": 66,
69
+ "C": 67,
70
+ "D": 68,
71
+ "E": 69,
72
+ "F": 70,
73
+ "G": 71,
74
+ "H": 72,
75
+ "I": 73,
76
+ "J": 74,
77
+ "K": 75,
78
+ "L": 76,
79
+ "M": 77,
80
+ "N": 78,
81
+ "O": 79,
82
+ "P": 80,
83
+ "Q": 81,
84
+ "R": 82,
85
+ "S": 83,
86
+ "T": 84,
87
+ "U": 85,
88
+ "V": 86,
89
+ "W": 87,
90
+ "X": 88,
91
+ "Y": 89,
92
+ "Z": 90,
93
+ "[": 91,
94
+ "\\": 92,
95
+ "]": 93,
96
+ "^": 94,
97
+ "_": 95,
98
+ "`": 96,
99
+ "a": 97,
100
+ "b": 98,
101
+ "c": 99,
102
+ "d": 100,
103
+ "e": 101,
104
+ "f": 102,
105
+ "g": 103,
106
+ "h": 104,
107
+ "i": 105,
108
+ "j": 106,
109
+ "k": 107,
110
+ "l": 108,
111
+ "m": 109,
112
+ "n": 110,
113
+ "o": 111,
114
+ "p": 112,
115
+ "q": 113,
116
+ "r": 114,
117
+ "s": 115,
118
+ "t": 116,
119
+ "u": 117,
120
+ "v": 118,
121
+ "w": 119,
122
+ "x": 120,
123
+ "y": 121,
124
+ "z": 122,
125
+ "{": 123,
126
+ "|": 124,
127
+ "}": 125,
128
+ "~": 126,
129
+ "\u007f": 127,
130
+ "\ufffd": 256,
131
+ ".\ufffd": 257,
132
+ "..": 258,
133
+ " b": 259,
134
+ " w": 260,
135
+ ".x": 261,
136
+ ".\u265f": 262,
137
+ ".\u2659": 263,
138
+ "\u265f": 264,
139
+ "\u2659": 265,
140
+ "\u2656": 266,
141
+ "\u265c": 267,
142
+ "\u2658": 268,
143
+ ".\u2658": 269,
144
+ "\u265e": 270,
145
+ ".\u265e": 271,
146
+ ".\u2656": 272,
147
+ ".\u265c": 273,
148
+ "\u2657": 274,
149
+ ".\u2657": 275,
150
+ "\u265d": 276,
151
+ ".\u265d": 277,
152
+ "\u265a": 278,
153
+ ".\u265a": 279,
154
+ "\u2654": 280,
155
+ ".\u2654": 281,
156
+ "\u2655": 282,
157
+ ".\u2655": 283,
158
+ "\u265b": 284,
159
+ ".\u265b": 285,
160
+ "..+": 286,
161
+ "\u265fd": 287,
162
+ "\u2659d": 288,
163
+ "\u2659e": 289,
164
+ "\u265fe": 290,
165
+ "\u265ag": 291,
166
+ "\u2654g": 292,
167
+ "\u265ef": 293,
168
+ "\u2658f": 294,
169
+ "\u265fc": 295,
170
+ "\u2656d": 296,
171
+ "\u2656f": 297,
172
+ "\u265cf": 298,
173
+ "\u265ed": 299,
174
+ "\u2659c": 300,
175
+ "\u2658d": 301,
176
+ "\u2658c": 302,
177
+ "\u265cd": 303,
178
+ ".+": 304,
179
+ "\u265ec": 305,
180
+ "\u2656e": 306,
181
+ "\u2658e": 307,
182
+ "\u2657e": 308,
183
+ "\u265ee": 309,
184
+ "\u265de": 310,
185
+ "\u2659g": 311,
186
+ "\u2659f": 312,
187
+ "\u265cc": 313,
188
+ "\u265fg": 314,
189
+ "\u265fb": 315,
190
+ "\u265ce": 316,
191
+ "\u265ch": 317,
192
+ "\u2656h": 318,
193
+ "\u2657d": 319,
194
+ "\u2656c": 320,
195
+ "\u2659b": 321,
196
+ "\u265ff": 322,
197
+ "\u265dd": 323,
198
+ "\u2659h": 324,
199
+ "\u265fa": 325,
200
+ "\u2655d": 326,
201
+ "\u265af": 327,
202
+ "\u2659a": 328,
203
+ "\u265fh": 329,
204
+ "\u2654f": 330,
205
+ "\u2657f": 331,
206
+ "\u2655e": 332,
207
+ "\u2657g": 333,
208
+ "\u265bd": 334,
209
+ "\u265dg": 335,
210
+ "\u265df": 336,
211
+ "\u265bc": 337,
212
+ "\u265cb": 338,
213
+ "\u2657c": 339,
214
+ "\u2656a": 340,
215
+ "\u265ca": 341,
216
+ "\u265be": 342,
217
+ "\u2656b": 343,
218
+ "\u265dc": 344,
219
+ "\u265ae": 345,
220
+ "\u265db": 346,
221
+ "\u2655c": 347,
222
+ "\u2654e": 348,
223
+ "\u2657b": 349,
224
+ "\u2655f": 350,
225
+ "\u265bb": 351,
226
+ "\u265ah": 352,
227
+ "\u2654h": 353,
228
+ "\u265ad": 354,
229
+ "\u265bf": 355,
230
+ "\u2654d": 356,
231
+ "\u2654c": 357,
232
+ "\u2655b": 358,
233
+ "\u265cg": 359,
234
+ "\u2656g": 360,
235
+ "\u265ac": 361,
236
+ "\u265eb": 362,
237
+ "\u2658g": 363,
238
+ "\u2655g": 364,
239
+ "\u2658b": 365,
240
+ "\u265eg": 366,
241
+ "\u265ba": 367,
242
+ "\u265bg": 368,
243
+ "\u2655h": 369,
244
+ "\u2654b": 370,
245
+ "\u2657h": 371,
246
+ "\u2655a": 372,
247
+ "\u265bh": 373,
248
+ "\u265da": 374,
249
+ "\u265ea": 375,
250
+ "\u265ab": 376,
251
+ "\u2657a": 377,
252
+ "\u265dh": 378,
253
+ "\u265eh": 379,
254
+ "\u2658a": 380,
255
+ "\u2658h": 381,
256
+ "\u2654a": 382,
257
+ "\u265aa": 383,
258
+ "..+#": 384,
259
+ "*.": 385,
260
+ ".+#": 386,
261
+ "*.+": 387
262
+ }