Hugging Face's logo

vldsavelyev
/
guitar_tab_gpt2_retrained

Text Generation
Transformers
PyTorch
gpt2
music
text-generation-inference
Model card Files
xet
 Community
2
vldsavelyev commited on
Commit
5e97881
·
1 Parent(s): 38f44e3

Upload tokenizer

Browse files
Files changed (4) hide show
  1. special_tokens_map.json +4 -1
  2. tokenizer.json +476 -466
  3. tokenizer_config.json +3 -0
  4. vocab.json +1 -1
special_tokens_map.json CHANGED
@@ -1,6 +1,9 @@
1
  {
 
 
 
2
  "bos_token": "<|endoftext|>",
3
  "eos_token": "<|endoftext|>",
4
- "pad_token": "<|endoftext|>",
5
  "unk_token": "<|endoftext|>"
6
  }
 
1
  {
2
+ "additional_special_tokens": [
3
+ "<|pad|>"
4
+ ],
5
  "bos_token": "<|endoftext|>",
6
  "eos_token": "<|endoftext|>",
7
+ "pad_token": "<|pad|>",
8
  "unk_token": "<|endoftext|>"
9
  }
tokenizer.json CHANGED
@@ -11,6 +11,15 @@
11
  "rstrip": false,
12
  "normalized": false,
13
  "special": true
 
 
 
 
 
 
 
 
 
14
  }
15
  ],
16
  "normalizer": null,
@@ -41,472 +50,473 @@
41
  "fuse_unk": false,
42
  "vocab": {
43
  "<|endoftext|>": 0,
44
- "!": 1,
45
- "\"": 2,
46
- "#": 3,
47
- "$": 4,
48
- "%": 5,
49
- "&": 6,
50
- "'": 7,
51
- "(": 8,
52
- ")": 9,
53
- "*": 10,
54
- "+": 11,
55
- ",": 12,
56
- "-": 13,
57
- ".": 14,
58
- "/": 15,
59
- "0": 16,
60
- "1": 17,
61
- "2": 18,
62
- "3": 19,
63
- "4": 20,
64
- "5": 21,
65
- "6": 22,
66
- "7": 23,
67
- "8": 24,
68
- "9": 25,
69
- ":": 26,
70
- ";": 27,
71
- "<": 28,
72
- "=": 29,
73
- ">": 30,
74
- "?": 31,
75
- "@": 32,
76
- "A": 33,
77
- "B": 34,
78
- "C": 35,
79
- "D": 36,
80
- "E": 37,
81
- "F": 38,
82
- "G": 39,
83
- "H": 40,
84
- "I": 41,
85
- "J": 42,
86
- "K": 43,
87
- "L": 44,
88
- "M": 45,
89
- "N": 46,
90
- "O": 47,
91
- "P": 48,
92
- "Q": 49,
93
- "R": 50,
94
- "S": 51,
95
- "T": 52,
96
- "U": 53,
97
- "V": 54,
98
- "W": 55,
99
- "X": 56,
100
- "Y": 57,
101
- "Z": 58,
102
- "[": 59,
103
- "\\": 60,
104
- "]": 61,
105
- "^": 62,
106
- "_": 63,
107
- "`": 64,
108
- "a": 65,
109
- "b": 66,
110
- "c": 67,
111
- "d": 68,
112
- "e": 69,
113
- "f": 70,
114
- "g": 71,
115
- "h": 72,
116
- "i": 73,
117
- "j": 74,
118
- "k": 75,
119
- "l": 76,
120
- "m": 77,
121
- "n": 78,
122
- "o": 79,
123
- "p": 80,
124
- "q": 81,
125
- "r": 82,
126
- "s": 83,
127
- "t": 84,
128
- "u": 85,
129
- "v": 86,
130
- "w": 87,
131
- "x": 88,
132
- "y": 89,
133
- "z": 90,
134
- "{": 91,
135
- "|": 92,
136
- "}": 93,
137
- "~": 94,
138
- "¡": 95,
139
- "¢": 96,
140
- "£": 97,
141
- "¤": 98,
142
- "¥": 99,
143
- "¦": 100,
144
- "§": 101,
145
- "¨": 102,
146
- "©": 103,
147
- "ª": 104,
148
- "«": 105,
149
- "¬": 106,
150
- "®": 107,
151
- "¯": 108,
152
- "°": 109,
153
- "±": 110,
154
- "²": 111,
155
- "³": 112,
156
- "´": 113,
157
- "µ": 114,
158
- "": 115,
159
- "·": 116,
160
- "¸": 117,
161
- "¹": 118,
162
- "º": 119,
163
- "»": 120,
164
- "¼": 121,
165
- "½": 122,
166
- "¾": 123,
167
- "¿": 124,
168
- "À": 125,
169
- "Á": 126,
170
- "Â": 127,
171
- "Ã": 128,
172
- "Ä": 129,
173
- "Å": 130,
174
- "Æ": 131,
175
- "Ç": 132,
176
- "È": 133,
177
- "É": 134,
178
- "Ê": 135,
179
- "Ë": 136,
180
- "Ì": 137,
181
- "Í": 138,
182
- "Î": 139,
183
- "Ï": 140,
184
- "Ð": 141,
185
- "Ñ": 142,
186
- "Ò": 143,
187
- "Ó": 144,
188
- "Ô": 145,
189
- "Õ": 146,
190
- "Ö": 147,
191
- "×": 148,
192
- "Ø": 149,
193
- "Ù": 150,
194
- "Ú": 151,
195
- "Û": 152,
196
- "Ü": 153,
197
- "Ý": 154,
198
- "Þ": 155,
199
- "ß": 156,
200
- "à": 157,
201
- "á": 158,
202
- "â": 159,
203
- "ã": 160,
204
- "ä": 161,
205
- "å": 162,
206
- "æ": 163,
207
- "ç": 164,
208
- "è": 165,
209
- "é": 166,
210
- "ê": 167,
211
- "ë": 168,
212
- "ì": 169,
213
- "í": 170,
214
- "î": 171,
215
- "ï": 172,
216
- "ð": 173,
217
- "ñ": 174,
218
- "ò": 175,
219
- "ó": 176,
220
- "ô": 177,
221
- "õ": 178,
222
- "ö": 179,
223
- "÷": 180,
224
- "ø": 181,
225
- "ù": 182,
226
- "ú": 183,
227
- "û": 184,
228
- "ü": 185,
229
- "ý": 186,
230
- "þ": 187,
231
- "ÿ": 188,
232
- "Ā": 189,
233
- "ā": 190,
234
- "Ă": 191,
235
- "ă": 192,
236
- "Ą": 193,
237
- "ą": 194,
238
- "Ć": 195,
239
- "ć": 196,
240
- "Ĉ": 197,
241
- "ĉ": 198,
242
- "Ċ": 199,
243
- "ċ": 200,
244
- "Č": 201,
245
- "č": 202,
246
- "Ď": 203,
247
- "ď": 204,
248
- "Đ": 205,
249
- "đ": 206,
250
- "Ē": 207,
251
- "ē": 208,
252
- "Ĕ": 209,
253
- "ĕ": 210,
254
- "Ė": 211,
255
- "ė": 212,
256
- "Ę": 213,
257
- "ę": 214,
258
- "Ě": 215,
259
- "ě": 216,
260
- "Ĝ": 217,
261
- "ĝ": 218,
262
- "Ğ": 219,
263
- "ğ": 220,
264
- "Ġ": 221,
265
- "ġ": 222,
266
- "Ģ": 223,
267
- "ģ": 224,
268
- "Ĥ": 225,
269
- "ĥ": 226,
270
- "Ħ": 227,
271
- "ħ": 228,
272
- "Ĩ": 229,
273
- "ĩ": 230,
274
- "Ī": 231,
275
- "ī": 232,
276
- "Ĭ": 233,
277
- "ĭ": 234,
278
- "Į": 235,
279
- "į": 236,
280
- "İ": 237,
281
- "ı": 238,
282
- "IJ": 239,
283
- "ij": 240,
284
- "Ĵ": 241,
285
- "ĵ": 242,
286
- "Ķ": 243,
287
- "ķ": 244,
288
- "ĸ": 245,
289
- "Ĺ": 246,
290
- "ĺ": 247,
291
- "Ļ": 248,
292
- "ļ": 249,
293
- "Ľ": 250,
294
- "ľ": 251,
295
- "Ŀ": 252,
296
- "ŀ": 253,
297
- "Ł": 254,
298
- "ł": 255,
299
- "Ń": 256,
300
- "Ġ(": 257,
301
- "Ġ3": 258,
302
- "Ġ0": 259,
303
- "Ġ|": 260,
304
- "Ġ4": 261,
305
- "Ġ1": 262,
306
- "Ġr": 263,
307
- "16": 264,
308
- "Ġ5": 265,
309
- "Ġ2": 266,
310
- ").": 267,
311
- "Ġ36": 268,
312
- "tu": 269,
313
- "Ġ7": 270,
314
- "Ġ:": 271,
315
- "Ġ6": 272,
316
- "}.": 273,
317
- "Ġ35": 274,
318
- "Ġ8": 275,
319
- "Ġ40": 276,
320
- "Ġ9": 277,
321
- "Ġ38": 278,
322
- "Ġ12": 279,
323
- "})": 280,
324
- "Ġ42": 281,
325
- "Ġ10": 282,
326
- "pm": 283,
327
- "lr": 284,
328
- "46": 285,
329
- "42": 286,
330
- "32": 287,
331
- "Ġ46": 288,
332
- "36": 289,
333
- "}).": 290,
334
- "49": 291,
335
- "Ġ14": 292,
336
- "Ġ11": 293,
337
- "Ġtu": 294,
338
- "Ġ15": 295,
339
- "51": 296,
340
- "Ġ44": 297,
341
- "10": 298,
342
- "Ġ51": 299,
343
- "57": 300,
344
- "38": 301,
345
- "Ġ13": 302,
346
- "Ġ17": 303,
347
- "40": 304,
348
- "be": 305,
349
- "44": 306,
350
- "12": 307,
351
- "35": 308,
352
- "Ġ49": 309,
353
- "Ġ45": 310,
354
- "Ġ57": 311,
355
- "Ġ41": 312,
356
- "Ġ43": 313,
357
- "Ġ16": 314,
358
- "Ġ19": 315,
359
- "11": 316,
360
- "Ġ54": 317,
361
- "53": 318,
362
- ")}": 319,
363
- "Ġ47": 320,
364
- "59": 321,
365
- "55": 322,
366
- "st": 323,
367
- "Ġ59": 324,
368
- "14": 325,
369
- "Ġ53": 326,
370
- "Ġ55": 327,
371
- "Ġ18": 328,
372
- "Ġ20": 329,
373
- "64": 330,
374
- ")}.": 331,
375
- "Ġ48": 332,
376
- "45": 333,
377
- "13": 334,
378
- "41": 335,
379
- "15": 336,
380
- "Ġ22": 337,
381
- "43": 338,
382
- "ah": 339,
383
- "Ġ69": 340,
384
- "Ġ82": 341,
385
- "Ġ21": 342,
386
- "17": 343,
387
- "47": 344,
388
- "Ġg": 345,
389
- "Ġbe": 346,
390
- "Ġv": 347,
391
- "Ġlr": 348,
392
- "Ġ37": 349,
393
- "54": 350,
394
- "52": 351,
395
- "Ġ50": 352,
396
- "Ġ24": 353,
397
- "Ġ70": 354,
398
- "nh": 355,
399
- "48": 356,
400
- "Ġ39": 357,
401
- "19": 358,
402
- "Ġ52": 359,
403
- ")})": 360,
404
- "Ġ64": 361,
405
- "20": 362,
406
- "Ġ63": 363,
407
- "69": 364,
408
- "Ġ61": 365,
409
- "Ġ23": 366,
410
- ")}).": 367,
411
- "Ġ27": 368,
412
- "Ġ33": 369,
413
- "82": 370,
414
- "18": 371,
415
- "22": 372,
416
- "Ġ60": 373,
417
- "Ġ28": 374,
418
- "50": 375,
419
- "Ġ31": 376,
420
- "70": 377,
421
- "21": 378,
422
- "Ġh": 379,
423
- "Ġ62": 380,
424
- "37": 381,
425
- "24": 382,
426
- "Ġ25": 383,
427
- "Ġf": 384,
428
- "Ġ56": 385,
429
- "Ġ81": 386,
430
- "Ġ87": 387,
431
- "Ġ29": 388,
432
- "Ġ26": 389,
433
- "39": 390,
434
- "Ġ85": 391,
435
- "Ġ65": 392,
436
- "Ġpm": 393,
437
- "Ġah": 394,
438
- "Ġ32": 395,
439
- "Ġ86": 396,
440
- "56": 397,
441
- "86": 398,
442
- "33": 399,
443
- "Ġ34": 400,
444
- "27": 401,
445
- "Ġ77": 402,
446
- "Ġ30": 403,
447
- "62": 404,
448
- "28": 405,
449
- "60": 406,
450
- "61": 407,
451
- "23": 408,
452
- "Ġ80": 409,
453
- "sh": 410,
454
- "Ġst": 411,
455
- "63": 412,
456
- "25": 413,
457
- "ph": 414,
458
- "Ġ75": 415,
459
- "87": 416,
460
- "Ġ99": 417,
461
- "31": 418,
462
- "Ġ76": 419,
463
- "Ġnh": 420,
464
- "26": 421,
465
- "65": 422,
466
- "Ġ74": 423,
467
- "Ġ66": 424,
468
- "tr": 425,
469
- "th": 426,
470
- "76": 427,
471
- "81": 428,
472
- "29": 429,
473
- "Ġ68": 430,
474
- "34": 431,
475
- "77": 432,
476
- "Ġ83": 433,
477
- "Ġ67": 434,
478
- "Ġ84": 435,
479
- "30": 436,
480
- "Ġph": 437,
481
- "80": 438,
482
- "84": 439,
483
- "Ġ73": 440,
484
- "73": 441,
485
- "75": 442,
486
- "Ġsh": 443,
487
- "74": 444,
488
- "68": 445,
489
- "58": 446,
490
- "Ġ58": 447,
491
- "66": 448,
492
- "Ġtr": 449,
493
- "Ġ71": 450,
494
- "Ġ91": 451,
495
- "99": 452,
496
- "Ġ96": 453,
497
- "67": 454,
498
- "Ġ79": 455,
499
- "85": 456,
500
- "Ġ78": 457,
501
- "83": 458,
502
- "Ġ72": 459,
503
- "Ġ94": 460,
504
- "Ġth": 461,
505
- "71": 462,
506
- "79": 463,
507
- "96": 464,
508
- "72": 465,
509
- "95": 466
 
510
  },
511
  "merges": [
512
  "Ġ (",
 
11
  "rstrip": false,
12
  "normalized": false,
13
  "special": true
14
+ },
15
+ {
16
+ "id": 1,
17
+ "content": "<|pad|>",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
  }
24
  ],
25
  "normalizer": null,
 
50
  "fuse_unk": false,
51
  "vocab": {
52
  "<|endoftext|>": 0,
53
+ "<|pad|>": 1,
54
+ "!": 2,
55
+ "\"": 3,
56
+ "#": 4,
57
+ "$": 5,
58
+ "%": 6,
59
+ "&": 7,
60
+ "'": 8,
61
+ "(": 9,
62
+ ")": 10,
63
+ "*": 11,
64
+ "+": 12,
65
+ ",": 13,
66
+ "-": 14,
67
+ ".": 15,
68
+ "/": 16,
69
+ "0": 17,
70
+ "1": 18,
71
+ "2": 19,
72
+ "3": 20,
73
+ "4": 21,
74
+ "5": 22,
75
+ "6": 23,
76
+ "7": 24,
77
+ "8": 25,
78
+ "9": 26,
79
+ ":": 27,
80
+ ";": 28,
81
+ "<": 29,
82
+ "=": 30,
83
+ ">": 31,
84
+ "?": 32,
85
+ "@": 33,
86
+ "A": 34,
87
+ "B": 35,
88
+ "C": 36,
89
+ "D": 37,
90
+ "E": 38,
91
+ "F": 39,
92
+ "G": 40,
93
+ "H": 41,
94
+ "I": 42,
95
+ "J": 43,
96
+ "K": 44,
97
+ "L": 45,
98
+ "M": 46,
99
+ "N": 47,
100
+ "O": 48,
101
+ "P": 49,
102
+ "Q": 50,
103
+ "R": 51,
104
+ "S": 52,
105
+ "T": 53,
106
+ "U": 54,
107
+ "V": 55,
108
+ "W": 56,
109
+ "X": 57,
110
+ "Y": 58,
111
+ "Z": 59,
112
+ "[": 60,
113
+ "\\": 61,
114
+ "]": 62,
115
+ "^": 63,
116
+ "_": 64,
117
+ "`": 65,
118
+ "a": 66,
119
+ "b": 67,
120
+ "c": 68,
121
+ "d": 69,
122
+ "e": 70,
123
+ "f": 71,
124
+ "g": 72,
125
+ "h": 73,
126
+ "i": 74,
127
+ "j": 75,
128
+ "k": 76,
129
+ "l": 77,
130
+ "m": 78,
131
+ "n": 79,
132
+ "o": 80,
133
+ "p": 81,
134
+ "q": 82,
135
+ "r": 83,
136
+ "s": 84,
137
+ "t": 85,
138
+ "u": 86,
139
+ "v": 87,
140
+ "w": 88,
141
+ "x": 89,
142
+ "y": 90,
143
+ "z": 91,
144
+ "{": 92,
145
+ "|": 93,
146
+ "}": 94,
147
+ "~": 95,
148
+ "¡": 96,
149
+ "¢": 97,
150
+ "£": 98,
151
+ "¤": 99,
152
+ "¥": 100,
153
+ "¦": 101,
154
+ "§": 102,
155
+ "¨": 103,
156
+ "©": 104,
157
+ "ª": 105,
158
+ "«": 106,
159
+ "¬": 107,
160
+ "®": 108,
161
+ "¯": 109,
162
+ "°": 110,
163
+ "±": 111,
164
+ "²": 112,
165
+ "³": 113,
166
+ "´": 114,
167
+ "µ": 115,
168
+ "": 116,
169
+ "·": 117,
170
+ "¸": 118,
171
+ "¹": 119,
172
+ "º": 120,
173
+ "»": 121,
174
+ "¼": 122,
175
+ "½": 123,
176
+ "¾": 124,
177
+ "¿": 125,
178
+ "À": 126,
179
+ "Á": 127,
180
+ "Â": 128,
181
+ "Ã": 129,
182
+ "Ä": 130,
183
+ "Å": 131,
184
+ "Æ": 132,
185
+ "Ç": 133,
186
+ "È": 134,
187
+ "É": 135,
188
+ "Ê": 136,
189
+ "Ë": 137,
190
+ "Ì": 138,
191
+ "Í": 139,
192
+ "Î": 140,
193
+ "Ï": 141,
194
+ "Ð": 142,
195
+ "Ñ": 143,
196
+ "Ò": 144,
197
+ "Ó": 145,
198
+ "Ô": 146,
199
+ "Õ": 147,
200
+ "Ö": 148,
201
+ "×": 149,
202
+ "Ø": 150,
203
+ "Ù": 151,
204
+ "Ú": 152,
205
+ "Û": 153,
206
+ "Ü": 154,
207
+ "Ý": 155,
208
+ "Þ": 156,
209
+ "ß": 157,
210
+ "à": 158,
211
+ "á": 159,
212
+ "â": 160,
213
+ "ã": 161,
214
+ "ä": 162,
215
+ "å": 163,
216
+ "æ": 164,
217
+ "ç": 165,
218
+ "è": 166,
219
+ "é": 167,
220
+ "ê": 168,
221
+ "ë": 169,
222
+ "ì": 170,
223
+ "í": 171,
224
+ "î": 172,
225
+ "ï": 173,
226
+ "ð": 174,
227
+ "ñ": 175,
228
+ "ò": 176,
229
+ "ó": 177,
230
+ "ô": 178,
231
+ "õ": 179,
232
+ "ö": 180,
233
+ "÷": 181,
234
+ "ø": 182,
235
+ "ù": 183,
236
+ "ú": 184,
237
+ "û": 185,
238
+ "ü": 186,
239
+ "ý": 187,
240
+ "þ": 188,
241
+ "ÿ": 189,
242
+ "Ā": 190,
243
+ "ā": 191,
244
+ "Ă": 192,
245
+ "ă": 193,
246
+ "Ą": 194,
247
+ "ą": 195,
248
+ "Ć": 196,
249
+ "ć": 197,
250
+ "Ĉ": 198,
251
+ "ĉ": 199,
252
+ "Ċ": 200,
253
+ "ċ": 201,
254
+ "Č": 202,
255
+ "č": 203,
256
+ "Ď": 204,
257
+ "ď": 205,
258
+ "Đ": 206,
259
+ "đ": 207,
260
+ "Ē": 208,
261
+ "ē": 209,
262
+ "Ĕ": 210,
263
+ "ĕ": 211,
264
+ "Ė": 212,
265
+ "ė": 213,
266
+ "Ę": 214,
267
+ "ę": 215,
268
+ "Ě": 216,
269
+ "ě": 217,
270
+ "Ĝ": 218,
271
+ "ĝ": 219,
272
+ "Ğ": 220,
273
+ "ğ": 221,
274
+ "Ġ": 222,
275
+ "ġ": 223,
276
+ "Ģ": 224,
277
+ "ģ": 225,
278
+ "Ĥ": 226,
279
+ "ĥ": 227,
280
+ "Ħ": 228,
281
+ "ħ": 229,
282
+ "Ĩ": 230,
283
+ "ĩ": 231,
284
+ "Ī": 232,
285
+ "ī": 233,
286
+ "Ĭ": 234,
287
+ "ĭ": 235,
288
+ "Į": 236,
289
+ "į": 237,
290
+ "İ": 238,
291
+ "ı": 239,
292
+ "IJ": 240,
293
+ "ij": 241,
294
+ "Ĵ": 242,
295
+ "ĵ": 243,
296
+ "Ķ": 244,
297
+ "ķ": 245,
298
+ "ĸ": 246,
299
+ "Ĺ": 247,
300
+ "ĺ": 248,
301
+ "Ļ": 249,
302
+ "ļ": 250,
303
+ "Ľ": 251,
304
+ "ľ": 252,
305
+ "Ŀ": 253,
306
+ "ŀ": 254,
307
+ "Ł": 255,
308
+ "ł": 256,
309
+ "Ń": 257,
310
+ "Ġ(": 258,
311
+ "Ġ3": 259,
312
+ "Ġ0": 260,
313
+ "Ġ|": 261,
314
+ "Ġ4": 262,
315
+ "Ġ1": 263,
316
+ "Ġr": 264,
317
+ "16": 265,
318
+ "Ġ5": 266,
319
+ "Ġ2": 267,
320
+ ").": 268,
321
+ "Ġ36": 269,
322
+ "tu": 270,
323
+ "Ġ7": 271,
324
+ "Ġ:": 272,
325
+ "Ġ6": 273,
326
+ "}.": 274,
327
+ "Ġ35": 275,
328
+ "Ġ8": 276,
329
+ "Ġ40": 277,
330
+ "Ġ9": 278,
331
+ "Ġ38": 279,
332
+ "Ġ12": 280,
333
+ "})": 281,
334
+ "Ġ42": 282,
335
+ "Ġ10": 283,
336
+ "pm": 284,
337
+ "lr": 285,
338
+ "46": 286,
339
+ "42": 287,
340
+ "32": 288,
341
+ "Ġ46": 289,
342
+ "36": 290,
343
+ "}).": 291,
344
+ "49": 292,
345
+ "Ġ14": 293,
346
+ "Ġ11": 294,
347
+ "Ġtu": 295,
348
+ "Ġ15": 296,
349
+ "51": 297,
350
+ "Ġ44": 298,
351
+ "10": 299,
352
+ "Ġ51": 300,
353
+ "57": 301,
354
+ "38": 302,
355
+ "Ġ13": 303,
356
+ "Ġ17": 304,
357
+ "40": 305,
358
+ "be": 306,
359
+ "44": 307,
360
+ "12": 308,
361
+ "35": 309,
362
+ "Ġ49": 310,
363
+ "Ġ45": 311,
364
+ "Ġ57": 312,
365
+ "Ġ41": 313,
366
+ "Ġ43": 314,
367
+ "Ġ16": 315,
368
+ "Ġ19": 316,
369
+ "11": 317,
370
+ "Ġ54": 318,
371
+ "53": 319,
372
+ ")}": 320,
373
+ "Ġ47": 321,
374
+ "59": 322,
375
+ "55": 323,
376
+ "st": 324,
377
+ "Ġ59": 325,
378
+ "14": 326,
379
+ "Ġ53": 327,
380
+ "Ġ55": 328,
381
+ "Ġ18": 329,
382
+ "Ġ20": 330,
383
+ "64": 331,
384
+ ")}.": 332,
385
+ "Ġ48": 333,
386
+ "45": 334,
387
+ "13": 335,
388
+ "41": 336,
389
+ "15": 337,
390
+ "Ġ22": 338,
391
+ "43": 339,
392
+ "ah": 340,
393
+ "Ġ69": 341,
394
+ "Ġ82": 342,
395
+ "Ġ21": 343,
396
+ "17": 344,
397
+ "47": 345,
398
+ "Ġg": 346,
399
+ "Ġbe": 347,
400
+ "Ġv": 348,
401
+ "Ġlr": 349,
402
+ "Ġ37": 350,
403
+ "54": 351,
404
+ "52": 352,
405
+ "Ġ50": 353,
406
+ "Ġ24": 354,
407
+ "Ġ70": 355,
408
+ "nh": 356,
409
+ "48": 357,
410
+ "Ġ39": 358,
411
+ "19": 359,
412
+ "Ġ52": 360,
413
+ ")})": 361,
414
+ "Ġ64": 362,
415
+ "20": 363,
416
+ "Ġ63": 364,
417
+ "69": 365,
418
+ "Ġ61": 366,
419
+ "Ġ23": 367,
420
+ ")}).": 368,
421
+ "Ġ27": 369,
422
+ "Ġ33": 370,
423
+ "82": 371,
424
+ "18": 372,
425
+ "22": 373,
426
+ "Ġ60": 374,
427
+ "Ġ28": 375,
428
+ "50": 376,
429
+ "Ġ31": 377,
430
+ "70": 378,
431
+ "21": 379,
432
+ "Ġh": 380,
433
+ "Ġ62": 381,
434
+ "37": 382,
435
+ "24": 383,
436
+ "Ġ25": 384,
437
+ "Ġf": 385,
438
+ "Ġ56": 386,
439
+ "Ġ81": 387,
440
+ "Ġ87": 388,
441
+ "Ġ29": 389,
442
+ "Ġ26": 390,
443
+ "39": 391,
444
+ "Ġ85": 392,
445
+ "Ġ65": 393,
446
+ "Ġpm": 394,
447
+ "Ġah": 395,
448
+ "Ġ32": 396,
449
+ "Ġ86": 397,
450
+ "56": 398,
451
+ "86": 399,
452
+ "33": 400,
453
+ "Ġ34": 401,
454
+ "27": 402,
455
+ "Ġ77": 403,
456
+ "Ġ30": 404,
457
+ "62": 405,
458
+ "28": 406,
459
+ "60": 407,
460
+ "61": 408,
461
+ "23": 409,
462
+ "Ġ80": 410,
463
+ "sh": 411,
464
+ "Ġst": 412,
465
+ "63": 413,
466
+ "25": 414,
467
+ "ph": 415,
468
+ "Ġ75": 416,
469
+ "87": 417,
470
+ "Ġ99": 418,
471
+ "31": 419,
472
+ "Ġ76": 420,
473
+ "Ġnh": 421,
474
+ "26": 422,
475
+ "65": 423,
476
+ "Ġ74": 424,
477
+ "Ġ66": 425,
478
+ "tr": 426,
479
+ "th": 427,
480
+ "76": 428,
481
+ "81": 429,
482
+ "29": 430,
483
+ "Ġ68": 431,
484
+ "34": 432,
485
+ "77": 433,
486
+ "Ġ83": 434,
487
+ "Ġ67": 435,
488
+ "Ġ84": 436,
489
+ "30": 437,
490
+ "Ġph": 438,
491
+ "80": 439,
492
+ "84": 440,
493
+ "Ġ73": 441,
494
+ "73": 442,
495
+ "75": 443,
496
+ "Ġsh": 444,
497
+ "74": 445,
498
+ "68": 446,
499
+ "58": 447,
500
+ "Ġ58": 448,
501
+ "66": 449,
502
+ "Ġtr": 450,
503
+ "Ġ71": 451,
504
+ "Ġ91": 452,
505
+ "99": 453,
506
+ "Ġ96": 454,
507
+ "67": 455,
508
+ "Ġ79": 456,
509
+ "85": 457,
510
+ "Ġ78": 458,
511
+ "83": 459,
512
+ "Ġ72": 460,
513
+ "Ġ94": 461,
514
+ "Ġth": 462,
515
+ "71": 463,
516
+ "79": 464,
517
+ "96": 465,
518
+ "72": 466,
519
+ "95": 467
520
  },
521
  "merges": [
522
  "Ġ (",
tokenizer_config.json CHANGED
@@ -1,5 +1,8 @@
1
  {
2
  "add_prefix_space": false,
 
 
 
3
  "bos_token": "<|endoftext|>",
4
  "eos_token": "<|endoftext|>",
5
  "model_max_length": 1024,
 
1
  {
2
  "add_prefix_space": false,
3
+ "additional_special_tokens": [
4
+ "<|pad|>"
5
+ ],
6
  "bos_token": "<|endoftext|>",
7
  "eos_token": "<|endoftext|>",
8
  "model_max_length": 1024,
vocab.json CHANGED
@@ -1 +1 @@
1
- {"<|endoftext|>":0,"!":1,"\"":2,"#":3,"$":4,"%":5,"&":6,"'":7,"(":8,")":9,"*":10,"+":11,",":12,"-":13,".":14,"/":15,"0":16,"1":17,"2":18,"3":19,"4":20,"5":21,"6":22,"7":23,"8":24,"9":25,":":26,";":27,"<":28,"=":29,">":30,"?":31,"@":32,"A":33,"B":34,"C":35,"D":36,"E":37,"F":38,"G":39,"H":40,"I":41,"J":42,"K":43,"L":44,"M":45,"N":46,"O":47,"P":48,"Q":49,"R":50,"S":51,"T":52,"U":53,"V":54,"W":55,"X":56,"Y":57,"Z":58,"[":59,"\\":60,"]":61,"^":62,"_":63,"`":64,"a":65,"b":66,"c":67,"d":68,"e":69,"f":70,"g":71,"h":72,"i":73,"j":74,"k":75,"l":76,"m":77,"n":78,"o":79,"p":80,"q":81,"r":82,"s":83,"t":84,"u":85,"v":86,"w":87,"x":88,"y":89,"z":90,"{":91,"|":92,"}":93,"~":94,"¡":95,"¢":96,"£":97,"¤":98,"¥":99,"¦":100,"§":101,"¨":102,"©":103,"ª":104,"«":105,"¬":106,"®":107,"¯":108,"°":109,"±":110,"²":111,"³":112,"´":113,"µ":114,"¶":115,"·":116,"¸":117,"¹":118,"º":119,"»":120,"¼":121,"½":122,"¾":123,"¿":124,"À":125,"Á":126,"Â":127,"Ã":128,"Ä":129,"Å":130,"Æ":131,"Ç":132,"È":133,"É":134,"Ê":135,"Ë":136,"Ì":137,"Í":138,"Î":139,"Ï":140,"Ð":141,"Ñ":142,"Ò":143,"Ó":144,"Ô":145,"Õ":146,"Ö":147,"×":148,"Ø":149,"Ù":150,"Ú":151,"Û":152,"Ü":153,"Ý":154,"Þ":155,"ß":156,"à":157,"á":158,"â":159,"ã":160,"ä":161,"å":162,"æ":163,"ç":164,"è":165,"é":166,"ê":167,"ë":168,"ì":169,"í":170,"î":171,"ï":172,"ð":173,"ñ":174,"ò":175,"ó":176,"ô":177,"õ":178,"ö":179,"÷":180,"ø":181,"ù":182,"ú":183,"û":184,"ü":185,"ý":186,"þ":187,"ÿ":188,"Ā":189,"ā":190,"Ă":191,"ă":192,"Ą":193,"ą":194,"Ć":195,"ć":196,"Ĉ":197,"ĉ":198,"Ċ":199,"ċ":200,"Č":201,"č":202,"Ď":203,"ď":204,"Đ":205,"đ":206,"Ē":207,"ē":208,"Ĕ":209,"ĕ":210,"Ė":211,"ė":212,"Ę":213,"ę":214,"Ě":215,"ě":216,"Ĝ":217,"ĝ":218,"Ğ":219,"ğ":220,"Ġ":221,"ġ":222,"Ģ":223,"ģ":224,"Ĥ":225,"ĥ":226,"Ħ":227,"ħ":228,"Ĩ":229,"ĩ":230,"Ī":231,"ī":232,"Ĭ":233,"ĭ":234,"Į":235,"į":236,"İ":237,"ı":238,"IJ":239,"ij":240,"Ĵ":241,"ĵ":242,"Ķ":243,"ķ":244,"ĸ":245,"Ĺ":246,"ĺ":247,"Ļ":248,"ļ":249,"Ľ":250,"ľ":251,"Ŀ":252,"ŀ":253,"Ł":254,"ł":255,"Ń":256,"Ġ(":257,"Ġ3":258,"Ġ0":259,"Ġ|":260,"Ġ4":261,"Ġ1":262,"Ġr":263,"16":264,"Ġ5":265,"Ġ2":266,").":267,"Ġ36":268,"tu":269,"Ġ7":270,"Ġ:":271,"Ġ6":272,"}.":273,"Ġ35":274,"Ġ8":275,"Ġ40":276,"Ġ9":277,"Ġ38":278,"Ġ12":279,"})":280,"Ġ42":281,"Ġ10":282,"pm":283,"lr":284,"46":285,"42":286,"32":287,"Ġ46":288,"36":289,"}).":290,"49":291,"Ġ14":292,"Ġ11":293,"Ġtu":294,"Ġ15":295,"51":296,"Ġ44":297,"10":298,"Ġ51":299,"57":300,"38":301,"Ġ13":302,"Ġ17":303,"40":304,"be":305,"44":306,"12":307,"35":308,"Ġ49":309,"Ġ45":310,"Ġ57":311,"Ġ41":312,"Ġ43":313,"Ġ16":314,"Ġ19":315,"11":316,"Ġ54":317,"53":318,")}":319,"Ġ47":320,"59":321,"55":322,"st":323,"Ġ59":324,"14":325,"Ġ53":326,"Ġ55":327,"Ġ18":328,"Ġ20":329,"64":330,")}.":331,"Ġ48":332,"45":333,"13":334,"41":335,"15":336,"Ġ22":337,"43":338,"ah":339,"Ġ69":340,"Ġ82":341,"Ġ21":342,"17":343,"47":344,"Ġg":345,"Ġbe":346,"Ġv":347,"Ġlr":348,"Ġ37":349,"54":350,"52":351,"Ġ50":352,"Ġ24":353,"Ġ70":354,"nh":355,"48":356,"Ġ39":357,"19":358,"Ġ52":359,")})":360,"Ġ64":361,"20":362,"Ġ63":363,"69":364,"Ġ61":365,"Ġ23":366,")}).":367,"Ġ27":368,"Ġ33":369,"82":370,"18":371,"22":372,"Ġ60":373,"Ġ28":374,"50":375,"Ġ31":376,"70":377,"21":378,"Ġh":379,"Ġ62":380,"37":381,"24":382,"Ġ25":383,"Ġf":384,"Ġ56":385,"Ġ81":386,"Ġ87":387,"Ġ29":388,"Ġ26":389,"39":390,"Ġ85":391,"Ġ65":392,"Ġpm":393,"Ġah":394,"Ġ32":395,"Ġ86":396,"56":397,"86":398,"33":399,"Ġ34":400,"27":401,"Ġ77":402,"Ġ30":403,"62":404,"28":405,"60":406,"61":407,"23":408,"Ġ80":409,"sh":410,"Ġst":411,"63":412,"25":413,"ph":414,"Ġ75":415,"87":416,"Ġ99":417,"31":418,"Ġ76":419,"Ġnh":420,"26":421,"65":422,"Ġ74":423,"Ġ66":424,"tr":425,"th":426,"76":427,"81":428,"29":429,"Ġ68":430,"34":431,"77":432,"Ġ83":433,"Ġ67":434,"Ġ84":435,"30":436,"Ġph":437,"80":438,"84":439,"Ġ73":440,"73":441,"75":442,"Ġsh":443,"74":444,"68":445,"58":446,"Ġ58":447,"66":448,"Ġtr":449,"Ġ71":450,"Ġ91":451,"99":452,"Ġ96":453,"67":454,"Ġ79":455,"85":456,"Ġ78":457,"83":458,"Ġ72":459,"Ġ94":460,"Ġth":461,"71":462,"79":463,"96":464,"72":465,"95":466}
 
1
+ {"<|endoftext|>":0,"<|pad|>":1,"!":2,"\"":3,"#":4,"$":5,"%":6,"&":7,"'":8,"(":9,")":10,"*":11,"+":12,",":13,"-":14,".":15,"/":16,"0":17,"1":18,"2":19,"3":20,"4":21,"5":22,"6":23,"7":24,"8":25,"9":26,":":27,";":28,"<":29,"=":30,">":31,"?":32,"@":33,"A":34,"B":35,"C":36,"D":37,"E":38,"F":39,"G":40,"H":41,"I":42,"J":43,"K":44,"L":45,"M":46,"N":47,"O":48,"P":49,"Q":50,"R":51,"S":52,"T":53,"U":54,"V":55,"W":56,"X":57,"Y":58,"Z":59,"[":60,"\\":61,"]":62,"^":63,"_":64,"`":65,"a":66,"b":67,"c":68,"d":69,"e":70,"f":71,"g":72,"h":73,"i":74,"j":75,"k":76,"l":77,"m":78,"n":79,"o":80,"p":81,"q":82,"r":83,"s":84,"t":85,"u":86,"v":87,"w":88,"x":89,"y":90,"z":91,"{":92,"|":93,"}":94,"~":95,"¡":96,"¢":97,"£":98,"¤":99,"¥":100,"¦":101,"§":102,"¨":103,"©":104,"ª":105,"«":106,"¬":107,"®":108,"¯":109,"°":110,"±":111,"²":112,"³":113,"´":114,"µ":115,"¶":116,"·":117,"¸":118,"¹":119,"º":120,"»":121,"¼":122,"½":123,"¾":124,"¿":125,"À":126,"Á":127,"Â":128,"Ã":129,"Ä":130,"Å":131,"Æ":132,"Ç":133,"È":134,"É":135,"Ê":136,"Ë":137,"Ì":138,"Í":139,"Î":140,"Ï":141,"Ð":142,"Ñ":143,"Ò":144,"Ó":145,"Ô":146,"Õ":147,"Ö":148,"×":149,"Ø":150,"Ù":151,"Ú":152,"Û":153,"Ü":154,"Ý":155,"Þ":156,"ß":157,"à":158,"á":159,"â":160,"ã":161,"ä":162,"å":163,"æ":164,"ç":165,"è":166,"é":167,"ê":168,"ë":169,"ì":170,"í":171,"î":172,"ï":173,"ð":174,"ñ":175,"ò":176,"ó":177,"ô":178,"õ":179,"ö":180,"÷":181,"ø":182,"ù":183,"ú":184,"û":185,"ü":186,"ý":187,"þ":188,"ÿ":189,"Ā":190,"ā":191,"Ă":192,"ă":193,"Ą":194,"ą":195,"Ć":196,"ć":197,"Ĉ":198,"ĉ":199,"Ċ":200,"ċ":201,"Č":202,"č":203,"Ď":204,"ď":205,"Đ":206,"đ":207,"Ē":208,"ē":209,"Ĕ":210,"ĕ":211,"Ė":212,"ė":213,"Ę":214,"ę":215,"Ě":216,"ě":217,"Ĝ":218,"ĝ":219,"Ğ":220,"ğ":221,"Ġ":222,"ġ":223,"Ģ":224,"ģ":225,"Ĥ":226,"ĥ":227,"Ħ":228,"ħ":229,"Ĩ":230,"ĩ":231,"Ī":232,"ī":233,"Ĭ":234,"ĭ":235,"Į":236,"į":237,"İ":238,"ı":239,"IJ":240,"ij":241,"Ĵ":242,"ĵ":243,"Ķ":244,"ķ":245,"ĸ":246,"Ĺ":247,"ĺ":248,"Ļ":249,"ļ":250,"Ľ":251,"ľ":252,"Ŀ":253,"ŀ":254,"Ł":255,"ł":256,"Ń":257,"Ġ(":258,"Ġ3":259,"Ġ0":260,"Ġ|":261,"Ġ4":262,"Ġ1":263,"Ġr":264,"16":265,"Ġ5":266,"Ġ2":267,").":268,"Ġ36":269,"tu":270,"Ġ7":271,"Ġ:":272,"Ġ6":273,"}.":274,"Ġ35":275,"Ġ8":276,"Ġ40":277,"Ġ9":278,"Ġ38":279,"Ġ12":280,"})":281,"Ġ42":282,"Ġ10":283,"pm":284,"lr":285,"46":286,"42":287,"32":288,"Ġ46":289,"36":290,"}).":291,"49":292,"Ġ14":293,"Ġ11":294,"Ġtu":295,"Ġ15":296,"51":297,"Ġ44":298,"10":299,"Ġ51":300,"57":301,"38":302,"Ġ13":303,"Ġ17":304,"40":305,"be":306,"44":307,"12":308,"35":309,"Ġ49":310,"Ġ45":311,"Ġ57":312,"Ġ41":313,"Ġ43":314,"Ġ16":315,"Ġ19":316,"11":317,"Ġ54":318,"53":319,")}":320,"Ġ47":321,"59":322,"55":323,"st":324,"Ġ59":325,"14":326,"Ġ53":327,"Ġ55":328,"Ġ18":329,"Ġ20":330,"64":331,")}.":332,"Ġ48":333,"45":334,"13":335,"41":336,"15":337,"Ġ22":338,"43":339,"ah":340,"Ġ69":341,"Ġ82":342,"Ġ21":343,"17":344,"47":345,"Ġg":346,"Ġbe":347,"Ġv":348,"Ġlr":349,"Ġ37":350,"54":351,"52":352,"Ġ50":353,"Ġ24":354,"Ġ70":355,"nh":356,"48":357,"Ġ39":358,"19":359,"Ġ52":360,")})":361,"Ġ64":362,"20":363,"Ġ63":364,"69":365,"Ġ61":366,"Ġ23":367,")}).":368,"Ġ27":369,"Ġ33":370,"82":371,"18":372,"22":373,"Ġ60":374,"Ġ28":375,"50":376,"Ġ31":377,"70":378,"21":379,"Ġh":380,"Ġ62":381,"37":382,"24":383,"Ġ25":384,"Ġf":385,"Ġ56":386,"Ġ81":387,"Ġ87":388,"Ġ29":389,"Ġ26":390,"39":391,"Ġ85":392,"Ġ65":393,"Ġpm":394,"Ġah":395,"Ġ32":396,"Ġ86":397,"56":398,"86":399,"33":400,"Ġ34":401,"27":402,"Ġ77":403,"Ġ30":404,"62":405,"28":406,"60":407,"61":408,"23":409,"Ġ80":410,"sh":411,"Ġst":412,"63":413,"25":414,"ph":415,"Ġ75":416,"87":417,"Ġ99":418,"31":419,"Ġ76":420,"Ġnh":421,"26":422,"65":423,"Ġ74":424,"Ġ66":425,"tr":426,"th":427,"76":428,"81":429,"29":430,"Ġ68":431,"34":432,"77":433,"Ġ83":434,"Ġ67":435,"Ġ84":436,"30":437,"Ġph":438,"80":439,"84":440,"Ġ73":441,"73":442,"75":443,"Ġsh":444,"74":445,"68":446,"58":447,"Ġ58":448,"66":449,"Ġtr":450,"Ġ71":451,"Ġ91":452,"99":453,"Ġ96":454,"67":455,"Ġ79":456,"85":457,"Ġ78":458,"83":459,"Ġ72":460,"Ġ94":461,"Ġth":462,"71":463,"79":464,"96":465,"72":466,"95":467}