vpalod commited on
Commit
ca4c739
·
verified ·
1 Parent(s): 46dd3c3

Upload folder using huggingface_hub

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +6 -0
  2. tokenizer.json +1006 -0
  3. tokenizer_config.json +48 -0
special_tokens_map.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|bos|>",
3
+ "eos_token": "<|eos|>",
4
+ "pad_token": "<|pad|>",
5
+ "unk_token": "<|unk|>"
6
+ }
tokenizer.json ADDED
@@ -0,0 +1,1006 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": {
5
+ "strategy": "BatchLongest",
6
+ "direction": "Right",
7
+ "pad_to_multiple_of": null,
8
+ "pad_id": 940,
9
+ "pad_type_id": 0,
10
+ "pad_token": "<|pad|>"
11
+ },
12
+ "added_tokens": [
13
+ {
14
+ "id": 940,
15
+ "content": "<|pad|>",
16
+ "single_word": false,
17
+ "lstrip": false,
18
+ "rstrip": false,
19
+ "normalized": false,
20
+ "special": true
21
+ },
22
+ {
23
+ "id": 941,
24
+ "content": "<|unk|>",
25
+ "single_word": false,
26
+ "lstrip": false,
27
+ "rstrip": false,
28
+ "normalized": false,
29
+ "special": true
30
+ },
31
+ {
32
+ "id": 942,
33
+ "content": "<|bos|>",
34
+ "single_word": false,
35
+ "lstrip": false,
36
+ "rstrip": false,
37
+ "normalized": false,
38
+ "special": true
39
+ },
40
+ {
41
+ "id": 943,
42
+ "content": "<|eos|>",
43
+ "single_word": false,
44
+ "lstrip": false,
45
+ "rstrip": false,
46
+ "normalized": false,
47
+ "special": true
48
+ }
49
+ ],
50
+ "normalizer": null,
51
+ "pre_tokenizer": {
52
+ "type": "Whitespace"
53
+ },
54
+ "post_processor": null,
55
+ "decoder": null,
56
+ "model": {
57
+ "type": "WordLevel",
58
+ "vocab": {
59
+ "c0": 0,
60
+ "c1": 1,
61
+ "c2": 2,
62
+ "c3": 3,
63
+ "c4": 4,
64
+ "c5": 5,
65
+ "c6": 6,
66
+ "c7": 7,
67
+ "c8": 8,
68
+ "c9": 9,
69
+ "c10": 10,
70
+ "c11": 11,
71
+ "c12": 12,
72
+ "c13": 13,
73
+ "c14": 14,
74
+ "c15": 15,
75
+ "c16": 16,
76
+ "c17": 17,
77
+ "c18": 18,
78
+ "c19": 19,
79
+ "c20": 20,
80
+ "c21": 21,
81
+ "c22": 22,
82
+ "c23": 23,
83
+ "c24": 24,
84
+ "c25": 25,
85
+ "c26": 26,
86
+ "c27": 27,
87
+ "c28": 28,
88
+ "c29": 29,
89
+ "c30": 30,
90
+ "c31": 31,
91
+ "c32": 32,
92
+ "c33": 33,
93
+ "c34": 34,
94
+ "c35": 35,
95
+ "c36": 36,
96
+ "c37": 37,
97
+ "c38": 38,
98
+ "c39": 39,
99
+ "c40": 40,
100
+ "c41": 41,
101
+ "c42": 42,
102
+ "c43": 43,
103
+ "c44": 44,
104
+ "c45": 45,
105
+ "c46": 46,
106
+ "c47": 47,
107
+ "c48": 48,
108
+ "c49": 49,
109
+ "c50": 50,
110
+ "c51": 51,
111
+ "c52": 52,
112
+ "c53": 53,
113
+ "c54": 54,
114
+ "c55": 55,
115
+ "c56": 56,
116
+ "c57": 57,
117
+ "c58": 58,
118
+ "c59": 59,
119
+ "c60": 60,
120
+ "c61": 61,
121
+ "c62": 62,
122
+ "c63": 63,
123
+ "c64": 64,
124
+ "c65": 65,
125
+ "c66": 66,
126
+ "c67": 67,
127
+ "c68": 68,
128
+ "c69": 69,
129
+ "c70": 70,
130
+ "c71": 71,
131
+ "c72": 72,
132
+ "c73": 73,
133
+ "c74": 74,
134
+ "c75": 75,
135
+ "c76": 76,
136
+ "c77": 77,
137
+ "c78": 78,
138
+ "c79": 79,
139
+ "c80": 80,
140
+ "c81": 81,
141
+ "c82": 82,
142
+ "c83": 83,
143
+ "c84": 84,
144
+ "c85": 85,
145
+ "c86": 86,
146
+ "c87": 87,
147
+ "c88": 88,
148
+ "c89": 89,
149
+ "c90": 90,
150
+ "c91": 91,
151
+ "c92": 92,
152
+ "c93": 93,
153
+ "c94": 94,
154
+ "c95": 95,
155
+ "c96": 96,
156
+ "c97": 97,
157
+ "c98": 98,
158
+ "c99": 99,
159
+ "c100": 100,
160
+ "c101": 101,
161
+ "c102": 102,
162
+ "c103": 103,
163
+ "c104": 104,
164
+ "c105": 105,
165
+ "c106": 106,
166
+ "c107": 107,
167
+ "c108": 108,
168
+ "c109": 109,
169
+ "c110": 110,
170
+ "c111": 111,
171
+ "c112": 112,
172
+ "c113": 113,
173
+ "c114": 114,
174
+ "c115": 115,
175
+ "c116": 116,
176
+ "c117": 117,
177
+ "c118": 118,
178
+ "c119": 119,
179
+ "c120": 120,
180
+ "c121": 121,
181
+ "c122": 122,
182
+ "c123": 123,
183
+ "c124": 124,
184
+ "c125": 125,
185
+ "c126": 126,
186
+ "c127": 127,
187
+ "c128": 128,
188
+ "c129": 129,
189
+ "c130": 130,
190
+ "c131": 131,
191
+ "c132": 132,
192
+ "c133": 133,
193
+ "c134": 134,
194
+ "c135": 135,
195
+ "c136": 136,
196
+ "c137": 137,
197
+ "c138": 138,
198
+ "c139": 139,
199
+ "c140": 140,
200
+ "c141": 141,
201
+ "c142": 142,
202
+ "c143": 143,
203
+ "c144": 144,
204
+ "c145": 145,
205
+ "c146": 146,
206
+ "c147": 147,
207
+ "c148": 148,
208
+ "c149": 149,
209
+ "c150": 150,
210
+ "c151": 151,
211
+ "c152": 152,
212
+ "c153": 153,
213
+ "c154": 154,
214
+ "c155": 155,
215
+ "c156": 156,
216
+ "c157": 157,
217
+ "c158": 158,
218
+ "c159": 159,
219
+ "c160": 160,
220
+ "c161": 161,
221
+ "c162": 162,
222
+ "c163": 163,
223
+ "c164": 164,
224
+ "c165": 165,
225
+ "c166": 166,
226
+ "c167": 167,
227
+ "c168": 168,
228
+ "c169": 169,
229
+ "c170": 170,
230
+ "c171": 171,
231
+ "c172": 172,
232
+ "c173": 173,
233
+ "c174": 174,
234
+ "c175": 175,
235
+ "c176": 176,
236
+ "c177": 177,
237
+ "c178": 178,
238
+ "c179": 179,
239
+ "c180": 180,
240
+ "c181": 181,
241
+ "c182": 182,
242
+ "c183": 183,
243
+ "c184": 184,
244
+ "c185": 185,
245
+ "c186": 186,
246
+ "c187": 187,
247
+ "c188": 188,
248
+ "c189": 189,
249
+ "c190": 190,
250
+ "c191": 191,
251
+ "c192": 192,
252
+ "c193": 193,
253
+ "c194": 194,
254
+ "c195": 195,
255
+ "c196": 196,
256
+ "c197": 197,
257
+ "c198": 198,
258
+ "c199": 199,
259
+ "c200": 200,
260
+ "c201": 201,
261
+ "c202": 202,
262
+ "c203": 203,
263
+ "c204": 204,
264
+ "c205": 205,
265
+ "c206": 206,
266
+ "c207": 207,
267
+ "c208": 208,
268
+ "c209": 209,
269
+ "c210": 210,
270
+ "c211": 211,
271
+ "c212": 212,
272
+ "c213": 213,
273
+ "c214": 214,
274
+ "c215": 215,
275
+ "c216": 216,
276
+ "c217": 217,
277
+ "c218": 218,
278
+ "c219": 219,
279
+ "c220": 220,
280
+ "c221": 221,
281
+ "c222": 222,
282
+ "c223": 223,
283
+ "c224": 224,
284
+ "c225": 225,
285
+ "c226": 226,
286
+ "c227": 227,
287
+ "c228": 228,
288
+ "c229": 229,
289
+ "c230": 230,
290
+ "c231": 231,
291
+ "c232": 232,
292
+ "c233": 233,
293
+ "c234": 234,
294
+ "c235": 235,
295
+ "c236": 236,
296
+ "c237": 237,
297
+ "c238": 238,
298
+ "c239": 239,
299
+ "c240": 240,
300
+ "c241": 241,
301
+ "c242": 242,
302
+ "c243": 243,
303
+ "c244": 244,
304
+ "c245": 245,
305
+ "c246": 246,
306
+ "c247": 247,
307
+ "c248": 248,
308
+ "c249": 249,
309
+ "c250": 250,
310
+ "c251": 251,
311
+ "c252": 252,
312
+ "c253": 253,
313
+ "c254": 254,
314
+ "c255": 255,
315
+ "c256": 256,
316
+ "c257": 257,
317
+ "c258": 258,
318
+ "c259": 259,
319
+ "c260": 260,
320
+ "c261": 261,
321
+ "c262": 262,
322
+ "c263": 263,
323
+ "c264": 264,
324
+ "c265": 265,
325
+ "c266": 266,
326
+ "c267": 267,
327
+ "c268": 268,
328
+ "c269": 269,
329
+ "c270": 270,
330
+ "c271": 271,
331
+ "c272": 272,
332
+ "c273": 273,
333
+ "c274": 274,
334
+ "c275": 275,
335
+ "c276": 276,
336
+ "c277": 277,
337
+ "c278": 278,
338
+ "c279": 279,
339
+ "c280": 280,
340
+ "c281": 281,
341
+ "c282": 282,
342
+ "c283": 283,
343
+ "c284": 284,
344
+ "c285": 285,
345
+ "c286": 286,
346
+ "c287": 287,
347
+ "c288": 288,
348
+ "c289": 289,
349
+ "c290": 290,
350
+ "c291": 291,
351
+ "c292": 292,
352
+ "c293": 293,
353
+ "c294": 294,
354
+ "c295": 295,
355
+ "c296": 296,
356
+ "c297": 297,
357
+ "c298": 298,
358
+ "c299": 299,
359
+ "c300": 300,
360
+ "c301": 301,
361
+ "c302": 302,
362
+ "c303": 303,
363
+ "c304": 304,
364
+ "c305": 305,
365
+ "c306": 306,
366
+ "c307": 307,
367
+ "c308": 308,
368
+ "c309": 309,
369
+ "c310": 310,
370
+ "c311": 311,
371
+ "c312": 312,
372
+ "c313": 313,
373
+ "c314": 314,
374
+ "c315": 315,
375
+ "c316": 316,
376
+ "c317": 317,
377
+ "c318": 318,
378
+ "c319": 319,
379
+ "c320": 320,
380
+ "c321": 321,
381
+ "c322": 322,
382
+ "c323": 323,
383
+ "c324": 324,
384
+ "c325": 325,
385
+ "c326": 326,
386
+ "c327": 327,
387
+ "c328": 328,
388
+ "c329": 329,
389
+ "c330": 330,
390
+ "c331": 331,
391
+ "c332": 332,
392
+ "c333": 333,
393
+ "c334": 334,
394
+ "c335": 335,
395
+ "c336": 336,
396
+ "c337": 337,
397
+ "c338": 338,
398
+ "c339": 339,
399
+ "c340": 340,
400
+ "c341": 341,
401
+ "c342": 342,
402
+ "c343": 343,
403
+ "c344": 344,
404
+ "c345": 345,
405
+ "c346": 346,
406
+ "c347": 347,
407
+ "c348": 348,
408
+ "c349": 349,
409
+ "c350": 350,
410
+ "c351": 351,
411
+ "c352": 352,
412
+ "c353": 353,
413
+ "c354": 354,
414
+ "c355": 355,
415
+ "c356": 356,
416
+ "c357": 357,
417
+ "c358": 358,
418
+ "c359": 359,
419
+ "c360": 360,
420
+ "c361": 361,
421
+ "c362": 362,
422
+ "c363": 363,
423
+ "c364": 364,
424
+ "c365": 365,
425
+ "c366": 366,
426
+ "c367": 367,
427
+ "c368": 368,
428
+ "c369": 369,
429
+ "c370": 370,
430
+ "c371": 371,
431
+ "c372": 372,
432
+ "c373": 373,
433
+ "c374": 374,
434
+ "c375": 375,
435
+ "c376": 376,
436
+ "c377": 377,
437
+ "c378": 378,
438
+ "c379": 379,
439
+ "c380": 380,
440
+ "c381": 381,
441
+ "c382": 382,
442
+ "c383": 383,
443
+ "c384": 384,
444
+ "c385": 385,
445
+ "c386": 386,
446
+ "c387": 387,
447
+ "c388": 388,
448
+ "c389": 389,
449
+ "c390": 390,
450
+ "c391": 391,
451
+ "c392": 392,
452
+ "c393": 393,
453
+ "c394": 394,
454
+ "c395": 395,
455
+ "c396": 396,
456
+ "c397": 397,
457
+ "c398": 398,
458
+ "c399": 399,
459
+ "c400": 400,
460
+ "c401": 401,
461
+ "c402": 402,
462
+ "c403": 403,
463
+ "c404": 404,
464
+ "c405": 405,
465
+ "c406": 406,
466
+ "c407": 407,
467
+ "c408": 408,
468
+ "c409": 409,
469
+ "c410": 410,
470
+ "c411": 411,
471
+ "c412": 412,
472
+ "c413": 413,
473
+ "c414": 414,
474
+ "c415": 415,
475
+ "c416": 416,
476
+ "c417": 417,
477
+ "c418": 418,
478
+ "c419": 419,
479
+ "c420": 420,
480
+ "c421": 421,
481
+ "c422": 422,
482
+ "c423": 423,
483
+ "c424": 424,
484
+ "c425": 425,
485
+ "c426": 426,
486
+ "c427": 427,
487
+ "c428": 428,
488
+ "c429": 429,
489
+ "c430": 430,
490
+ "c431": 431,
491
+ "c432": 432,
492
+ "c433": 433,
493
+ "c434": 434,
494
+ "c435": 435,
495
+ "c436": 436,
496
+ "c437": 437,
497
+ "c438": 438,
498
+ "c439": 439,
499
+ "c440": 440,
500
+ "c441": 441,
501
+ "c442": 442,
502
+ "c443": 443,
503
+ "c444": 444,
504
+ "c445": 445,
505
+ "c446": 446,
506
+ "c447": 447,
507
+ "c448": 448,
508
+ "c449": 449,
509
+ "c450": 450,
510
+ "c451": 451,
511
+ "c452": 452,
512
+ "c453": 453,
513
+ "c454": 454,
514
+ "c455": 455,
515
+ "c456": 456,
516
+ "c457": 457,
517
+ "c458": 458,
518
+ "c459": 459,
519
+ "c460": 460,
520
+ "c461": 461,
521
+ "c462": 462,
522
+ "c463": 463,
523
+ "c464": 464,
524
+ "c465": 465,
525
+ "c466": 466,
526
+ "c467": 467,
527
+ "c468": 468,
528
+ "c469": 469,
529
+ "c470": 470,
530
+ "c471": 471,
531
+ "c472": 472,
532
+ "c473": 473,
533
+ "c474": 474,
534
+ "c475": 475,
535
+ "c476": 476,
536
+ "c477": 477,
537
+ "c478": 478,
538
+ "c479": 479,
539
+ "c480": 480,
540
+ "c481": 481,
541
+ "c482": 482,
542
+ "c483": 483,
543
+ "c484": 484,
544
+ "c485": 485,
545
+ "c486": 486,
546
+ "c487": 487,
547
+ "c488": 488,
548
+ "c489": 489,
549
+ "c490": 490,
550
+ "c491": 491,
551
+ "c492": 492,
552
+ "c493": 493,
553
+ "c494": 494,
554
+ "c495": 495,
555
+ "c496": 496,
556
+ "c497": 497,
557
+ "c498": 498,
558
+ "c499": 499,
559
+ "c500": 500,
560
+ "c501": 501,
561
+ "c502": 502,
562
+ "c503": 503,
563
+ "c504": 504,
564
+ "c505": 505,
565
+ "c506": 506,
566
+ "c507": 507,
567
+ "c508": 508,
568
+ "c509": 509,
569
+ "c510": 510,
570
+ "c511": 511,
571
+ "c512": 512,
572
+ "c513": 513,
573
+ "c514": 514,
574
+ "c515": 515,
575
+ "c516": 516,
576
+ "c517": 517,
577
+ "c518": 518,
578
+ "c519": 519,
579
+ "c520": 520,
580
+ "c521": 521,
581
+ "c522": 522,
582
+ "c523": 523,
583
+ "c524": 524,
584
+ "c525": 525,
585
+ "c526": 526,
586
+ "c527": 527,
587
+ "c528": 528,
588
+ "c529": 529,
589
+ "c530": 530,
590
+ "c531": 531,
591
+ "c532": 532,
592
+ "c533": 533,
593
+ "c534": 534,
594
+ "c535": 535,
595
+ "c536": 536,
596
+ "c537": 537,
597
+ "c538": 538,
598
+ "c539": 539,
599
+ "c540": 540,
600
+ "c541": 541,
601
+ "c542": 542,
602
+ "c543": 543,
603
+ "c544": 544,
604
+ "c545": 545,
605
+ "c546": 546,
606
+ "c547": 547,
607
+ "c548": 548,
608
+ "c549": 549,
609
+ "c550": 550,
610
+ "c551": 551,
611
+ "c552": 552,
612
+ "c553": 553,
613
+ "c554": 554,
614
+ "c555": 555,
615
+ "c556": 556,
616
+ "c557": 557,
617
+ "c558": 558,
618
+ "c559": 559,
619
+ "c560": 560,
620
+ "c561": 561,
621
+ "c562": 562,
622
+ "c563": 563,
623
+ "c564": 564,
624
+ "c565": 565,
625
+ "c566": 566,
626
+ "c567": 567,
627
+ "c568": 568,
628
+ "c569": 569,
629
+ "c570": 570,
630
+ "c571": 571,
631
+ "c572": 572,
632
+ "c573": 573,
633
+ "c574": 574,
634
+ "c575": 575,
635
+ "c576": 576,
636
+ "c577": 577,
637
+ "c578": 578,
638
+ "c579": 579,
639
+ "c580": 580,
640
+ "c581": 581,
641
+ "c582": 582,
642
+ "c583": 583,
643
+ "c584": 584,
644
+ "c585": 585,
645
+ "c586": 586,
646
+ "c587": 587,
647
+ "c588": 588,
648
+ "c589": 589,
649
+ "c590": 590,
650
+ "c591": 591,
651
+ "c592": 592,
652
+ "c593": 593,
653
+ "c594": 594,
654
+ "c595": 595,
655
+ "c596": 596,
656
+ "c597": 597,
657
+ "c598": 598,
658
+ "c599": 599,
659
+ "c600": 600,
660
+ "c601": 601,
661
+ "c602": 602,
662
+ "c603": 603,
663
+ "c604": 604,
664
+ "c605": 605,
665
+ "c606": 606,
666
+ "c607": 607,
667
+ "c608": 608,
668
+ "c609": 609,
669
+ "c610": 610,
670
+ "c611": 611,
671
+ "c612": 612,
672
+ "c613": 613,
673
+ "c614": 614,
674
+ "c615": 615,
675
+ "c616": 616,
676
+ "c617": 617,
677
+ "c618": 618,
678
+ "c619": 619,
679
+ "c620": 620,
680
+ "c621": 621,
681
+ "c622": 622,
682
+ "c623": 623,
683
+ "c624": 624,
684
+ "c625": 625,
685
+ "c626": 626,
686
+ "c627": 627,
687
+ "c628": 628,
688
+ "c629": 629,
689
+ "c630": 630,
690
+ "c631": 631,
691
+ "c632": 632,
692
+ "c633": 633,
693
+ "c634": 634,
694
+ "c635": 635,
695
+ "c636": 636,
696
+ "c637": 637,
697
+ "c638": 638,
698
+ "c639": 639,
699
+ "c640": 640,
700
+ "c641": 641,
701
+ "c642": 642,
702
+ "c643": 643,
703
+ "c644": 644,
704
+ "c645": 645,
705
+ "c646": 646,
706
+ "c647": 647,
707
+ "c648": 648,
708
+ "c649": 649,
709
+ "c650": 650,
710
+ "c651": 651,
711
+ "c652": 652,
712
+ "c653": 653,
713
+ "c654": 654,
714
+ "c655": 655,
715
+ "c656": 656,
716
+ "c657": 657,
717
+ "c658": 658,
718
+ "c659": 659,
719
+ "c660": 660,
720
+ "c661": 661,
721
+ "c662": 662,
722
+ "c663": 663,
723
+ "c664": 664,
724
+ "c665": 665,
725
+ "c666": 666,
726
+ "c667": 667,
727
+ "c668": 668,
728
+ "c669": 669,
729
+ "c670": 670,
730
+ "c671": 671,
731
+ "c672": 672,
732
+ "c673": 673,
733
+ "c674": 674,
734
+ "c675": 675,
735
+ "c676": 676,
736
+ "c677": 677,
737
+ "c678": 678,
738
+ "c679": 679,
739
+ "c680": 680,
740
+ "c681": 681,
741
+ "c682": 682,
742
+ "c683": 683,
743
+ "c684": 684,
744
+ "c685": 685,
745
+ "c686": 686,
746
+ "c687": 687,
747
+ "c688": 688,
748
+ "c689": 689,
749
+ "c690": 690,
750
+ "c691": 691,
751
+ "c692": 692,
752
+ "c693": 693,
753
+ "c694": 694,
754
+ "c695": 695,
755
+ "c696": 696,
756
+ "c697": 697,
757
+ "c698": 698,
758
+ "c699": 699,
759
+ "c700": 700,
760
+ "c701": 701,
761
+ "c702": 702,
762
+ "c703": 703,
763
+ "c704": 704,
764
+ "c705": 705,
765
+ "c706": 706,
766
+ "c707": 707,
767
+ "c708": 708,
768
+ "c709": 709,
769
+ "c710": 710,
770
+ "c711": 711,
771
+ "c712": 712,
772
+ "c713": 713,
773
+ "c714": 714,
774
+ "c715": 715,
775
+ "c716": 716,
776
+ "c717": 717,
777
+ "c718": 718,
778
+ "c719": 719,
779
+ "c720": 720,
780
+ "c721": 721,
781
+ "c722": 722,
782
+ "c723": 723,
783
+ "c724": 724,
784
+ "c725": 725,
785
+ "c726": 726,
786
+ "c727": 727,
787
+ "c728": 728,
788
+ "c729": 729,
789
+ "c730": 730,
790
+ "c731": 731,
791
+ "c732": 732,
792
+ "c733": 733,
793
+ "c734": 734,
794
+ "c735": 735,
795
+ "c736": 736,
796
+ "c737": 737,
797
+ "c738": 738,
798
+ "c739": 739,
799
+ "c740": 740,
800
+ "c741": 741,
801
+ "c742": 742,
802
+ "c743": 743,
803
+ "c744": 744,
804
+ "c745": 745,
805
+ "c746": 746,
806
+ "c747": 747,
807
+ "c748": 748,
808
+ "c749": 749,
809
+ "c750": 750,
810
+ "c751": 751,
811
+ "c752": 752,
812
+ "c753": 753,
813
+ "c754": 754,
814
+ "c755": 755,
815
+ "c756": 756,
816
+ "c757": 757,
817
+ "c758": 758,
818
+ "c759": 759,
819
+ "c760": 760,
820
+ "c761": 761,
821
+ "c762": 762,
822
+ "c763": 763,
823
+ "c764": 764,
824
+ "c765": 765,
825
+ "c766": 766,
826
+ "c767": 767,
827
+ "c768": 768,
828
+ "c769": 769,
829
+ "c770": 770,
830
+ "c771": 771,
831
+ "c772": 772,
832
+ "c773": 773,
833
+ "c774": 774,
834
+ "c775": 775,
835
+ "c776": 776,
836
+ "c777": 777,
837
+ "c778": 778,
838
+ "c779": 779,
839
+ "c780": 780,
840
+ "c781": 781,
841
+ "c782": 782,
842
+ "c783": 783,
843
+ "c784": 784,
844
+ "c785": 785,
845
+ "c786": 786,
846
+ "c787": 787,
847
+ "c788": 788,
848
+ "c789": 789,
849
+ "c790": 790,
850
+ "c791": 791,
851
+ "c792": 792,
852
+ "c793": 793,
853
+ "c794": 794,
854
+ "c795": 795,
855
+ "c796": 796,
856
+ "c797": 797,
857
+ "c798": 798,
858
+ "c799": 799,
859
+ "c800": 800,
860
+ "c801": 801,
861
+ "c802": 802,
862
+ "c803": 803,
863
+ "c804": 804,
864
+ "c805": 805,
865
+ "c806": 806,
866
+ "c807": 807,
867
+ "c808": 808,
868
+ "c809": 809,
869
+ "c810": 810,
870
+ "c811": 811,
871
+ "c812": 812,
872
+ "c813": 813,
873
+ "c814": 814,
874
+ "c815": 815,
875
+ "c816": 816,
876
+ "c817": 817,
877
+ "c818": 818,
878
+ "c819": 819,
879
+ "c820": 820,
880
+ "c821": 821,
881
+ "c822": 822,
882
+ "c823": 823,
883
+ "c824": 824,
884
+ "c825": 825,
885
+ "c826": 826,
886
+ "c827": 827,
887
+ "c828": 828,
888
+ "c829": 829,
889
+ "c830": 830,
890
+ "c831": 831,
891
+ "c832": 832,
892
+ "c833": 833,
893
+ "c834": 834,
894
+ "c835": 835,
895
+ "c836": 836,
896
+ "c837": 837,
897
+ "c838": 838,
898
+ "c839": 839,
899
+ "c840": 840,
900
+ "c841": 841,
901
+ "c842": 842,
902
+ "c843": 843,
903
+ "c844": 844,
904
+ "c845": 845,
905
+ "c846": 846,
906
+ "c847": 847,
907
+ "c848": 848,
908
+ "c849": 849,
909
+ "c850": 850,
910
+ "c851": 851,
911
+ "c852": 852,
912
+ "c853": 853,
913
+ "c854": 854,
914
+ "c855": 855,
915
+ "c856": 856,
916
+ "c857": 857,
917
+ "c858": 858,
918
+ "c859": 859,
919
+ "c860": 860,
920
+ "c861": 861,
921
+ "c862": 862,
922
+ "c863": 863,
923
+ "c864": 864,
924
+ "c865": 865,
925
+ "c866": 866,
926
+ "c867": 867,
927
+ "c868": 868,
928
+ "c869": 869,
929
+ "c870": 870,
930
+ "c871": 871,
931
+ "c872": 872,
932
+ "c873": 873,
933
+ "c874": 874,
934
+ "c875": 875,
935
+ "c876": 876,
936
+ "c877": 877,
937
+ "c878": 878,
938
+ "c879": 879,
939
+ "c880": 880,
940
+ "c881": 881,
941
+ "c882": 882,
942
+ "c883": 883,
943
+ "c884": 884,
944
+ "c885": 885,
945
+ "c886": 886,
946
+ "c887": 887,
947
+ "c888": 888,
948
+ "c889": 889,
949
+ "c890": 890,
950
+ "c891": 891,
951
+ "c892": 892,
952
+ "c893": 893,
953
+ "c894": 894,
954
+ "c895": 895,
955
+ "c896": 896,
956
+ "c897": 897,
957
+ "c898": 898,
958
+ "c899": 899,
959
+ "0": 900,
960
+ "1": 901,
961
+ "2": 902,
962
+ "3": 903,
963
+ "4": 904,
964
+ "5": 905,
965
+ "6": 906,
966
+ "7": 907,
967
+ "8": 908,
968
+ "9": 909,
969
+ "10": 910,
970
+ "11": 911,
971
+ "12": 912,
972
+ "13": 913,
973
+ "14": 914,
974
+ "15": 915,
975
+ "16": 916,
976
+ "17": 917,
977
+ "18": 918,
978
+ "19": 919,
979
+ "20": 920,
980
+ "21": 921,
981
+ "22": 922,
982
+ "23": 923,
983
+ "24": 924,
984
+ "25": 925,
985
+ "26": 926,
986
+ "27": 927,
987
+ "28": 928,
988
+ "29": 929,
989
+ "start": 930,
990
+ "goal": 931,
991
+ "wall": 932,
992
+ "create": 933,
993
+ "close": 934,
994
+ "plan": 935,
995
+ "query": 936,
996
+ "reasoning": 937,
997
+ "solution": 938,
998
+ "end": 939,
999
+ "<|pad|>": 940,
1000
+ "<|unk|>": 941,
1001
+ "<|bos|>": 942,
1002
+ "<|eos|>": 943
1003
+ },
1004
+ "unk_token": "<|unk|>"
1005
+ }
1006
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "940": {
4
+ "content": "<|pad|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "941": {
12
+ "content": "<|unk|>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "942": {
20
+ "content": "<|bos|>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "943": {
28
+ "content": "<|eos|>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ }
35
+ },
36
+ "bos_token": "<|bos|>",
37
+ "clean_up_tokenization_spaces": false,
38
+ "eos_token": "<|eos|>",
39
+ "extra_special_tokens": {},
40
+ "max_length": null,
41
+ "model_max_length": 1000000000000000019884624838656,
42
+ "pad_to_multiple_of": null,
43
+ "pad_token": "<|pad|>",
44
+ "pad_token_type_id": 0,
45
+ "padding_side": "right",
46
+ "tokenizer_class": "PreTrainedTokenizer",
47
+ "unk_token": "<|unk|>"
48
+ }