Hugging Face's logo

TylerHMC
/
GigaMIDI_tokeniser

Model card Files
xet
 Community
TylerHMC commited on
Commit
94ce729
·
verified ·
1 Parent(s): 7282cc6

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. tokenizer.json +444 -0
  2. tokenizer_metadata.json +167 -0
tokenizer.json ADDED
@@ -0,0 +1,444 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "pitch_range": [
4
+ 21,
5
+ 109
6
+ ],
7
+ "beat_res": {
8
+ "0_4": 8,
9
+ "4_12": 4
10
+ },
11
+ "num_velocities": 32,
12
+ "remove_duplicated_notes": true,
13
+ "encode_ids_split": "bar",
14
+ "special_tokens": [
15
+ "PAD_None",
16
+ "BOS_None",
17
+ "EOS_None",
18
+ "MASK_None"
19
+ ],
20
+ "use_velocities": false,
21
+ "use_note_duration_programs": [
22
+ 0,
23
+ 1,
24
+ 2,
25
+ 3,
26
+ 4,
27
+ 5,
28
+ 6,
29
+ 7,
30
+ 8,
31
+ 9,
32
+ 10,
33
+ 11,
34
+ 12,
35
+ 13,
36
+ 14,
37
+ 15,
38
+ 16,
39
+ 17,
40
+ 18,
41
+ 19,
42
+ 20,
43
+ 21,
44
+ 22,
45
+ 23,
46
+ 24,
47
+ 25,
48
+ 26,
49
+ 27,
50
+ 28,
51
+ 29,
52
+ 30,
53
+ 31,
54
+ 32,
55
+ 33,
56
+ 34,
57
+ 35,
58
+ 36,
59
+ 37,
60
+ 38,
61
+ 39,
62
+ 40,
63
+ 41,
64
+ 42,
65
+ 43,
66
+ 44,
67
+ 45,
68
+ 46,
69
+ 47,
70
+ 48,
71
+ 49,
72
+ 50,
73
+ 51,
74
+ 52,
75
+ 53,
76
+ 54,
77
+ 55,
78
+ 56,
79
+ 57,
80
+ 58,
81
+ 59,
82
+ 60,
83
+ 61,
84
+ 62,
85
+ 63,
86
+ 64,
87
+ 65,
88
+ 66,
89
+ 67,
90
+ 68,
91
+ 69,
92
+ 70,
93
+ 71,
94
+ 72,
95
+ 73,
96
+ 74,
97
+ 75,
98
+ 76,
99
+ 77,
100
+ 78,
101
+ 79,
102
+ 80,
103
+ 81,
104
+ 82,
105
+ 83,
106
+ 84,
107
+ 85,
108
+ 86,
109
+ 87,
110
+ 88,
111
+ 89,
112
+ 90,
113
+ 91,
114
+ 92,
115
+ 93,
116
+ 94,
117
+ 95,
118
+ 96,
119
+ 97,
120
+ 98,
121
+ 99,
122
+ 100,
123
+ 101,
124
+ 102,
125
+ 103,
126
+ 104,
127
+ 105,
128
+ 106,
129
+ 107,
130
+ 108,
131
+ 109,
132
+ 110,
133
+ 111,
134
+ 112,
135
+ 113,
136
+ 114,
137
+ 115,
138
+ 116,
139
+ 117,
140
+ 118,
141
+ 119,
142
+ 120,
143
+ 121,
144
+ 122,
145
+ 123,
146
+ 124,
147
+ 125,
148
+ 126,
149
+ 127,
150
+ -1
151
+ ],
152
+ "use_chords": true,
153
+ "use_rests": false,
154
+ "use_tempos": true,
155
+ "use_time_signatures": true,
156
+ "use_sustain_pedals": false,
157
+ "use_pitch_bends": false,
158
+ "use_programs": true,
159
+ "use_pitch_intervals": false,
160
+ "use_pitchdrum_tokens": true,
161
+ "default_note_duration": 0.5,
162
+ "programs": [
163
+ 0,
164
+ 1,
165
+ 2,
166
+ 3,
167
+ 4,
168
+ 5,
169
+ 6,
170
+ 7,
171
+ 8,
172
+ 9,
173
+ 10,
174
+ 11,
175
+ 12,
176
+ 13,
177
+ 14,
178
+ 15,
179
+ 16,
180
+ 17,
181
+ 18,
182
+ 19,
183
+ 20,
184
+ 21,
185
+ 22,
186
+ 23,
187
+ 24,
188
+ 25,
189
+ 26,
190
+ 27,
191
+ 28,
192
+ 29,
193
+ 30,
194
+ 31,
195
+ 32,
196
+ 33,
197
+ 34,
198
+ 35,
199
+ 36,
200
+ 37,
201
+ 38,
202
+ 39,
203
+ 40,
204
+ 41,
205
+ 42,
206
+ 43,
207
+ 44,
208
+ 45,
209
+ 46,
210
+ 47,
211
+ 48,
212
+ 49,
213
+ 50,
214
+ 51,
215
+ 52,
216
+ 53,
217
+ 54,
218
+ 55,
219
+ 56,
220
+ 57,
221
+ 58,
222
+ 59,
223
+ 60,
224
+ 61,
225
+ 62,
226
+ 63,
227
+ 64,
228
+ 65,
229
+ 66,
230
+ 67,
231
+ 68,
232
+ 69,
233
+ 70,
234
+ 71,
235
+ 72,
236
+ 73,
237
+ 74,
238
+ 75,
239
+ 76,
240
+ 77,
241
+ 78,
242
+ 79,
243
+ 80,
244
+ 81,
245
+ 82,
246
+ 83,
247
+ 84,
248
+ 85,
249
+ 86,
250
+ 87,
251
+ 88,
252
+ 89,
253
+ 90,
254
+ 91,
255
+ 92,
256
+ 93,
257
+ 94,
258
+ 95,
259
+ 96,
260
+ 97,
261
+ 98,
262
+ 99,
263
+ 100,
264
+ 101,
265
+ 102,
266
+ 103,
267
+ 104,
268
+ 105,
269
+ 106,
270
+ 107,
271
+ 108,
272
+ 109,
273
+ 110,
274
+ 111,
275
+ 112,
276
+ 113,
277
+ 114,
278
+ 115,
279
+ 116,
280
+ 117,
281
+ 118,
282
+ 119,
283
+ 120,
284
+ 121,
285
+ 122,
286
+ 123,
287
+ 124,
288
+ 125,
289
+ 126,
290
+ 127,
291
+ -1
292
+ ],
293
+ "one_token_stream_for_programs": true,
294
+ "program_changes": true,
295
+ "beat_res_rest": {
296
+ "0_1": 8,
297
+ "1_2": 4,
298
+ "2_12": 2
299
+ },
300
+ "chord_maps": {
301
+ "min": [
302
+ 0,
303
+ 3,
304
+ 7
305
+ ],
306
+ "maj": [
307
+ 0,
308
+ 4,
309
+ 7
310
+ ],
311
+ "dim": [
312
+ 0,
313
+ 3,
314
+ 6
315
+ ],
316
+ "aug": [
317
+ 0,
318
+ 4,
319
+ 8
320
+ ],
321
+ "sus2": [
322
+ 0,
323
+ 2,
324
+ 7
325
+ ],
326
+ "sus4": [
327
+ 0,
328
+ 5,
329
+ 7
330
+ ],
331
+ "7dom": [
332
+ 0,
333
+ 4,
334
+ 7,
335
+ 10
336
+ ],
337
+ "7min": [
338
+ 0,
339
+ 3,
340
+ 7,
341
+ 10
342
+ ],
343
+ "7maj": [
344
+ 0,
345
+ 4,
346
+ 7,
347
+ 11
348
+ ],
349
+ "7halfdim": [
350
+ 0,
351
+ 3,
352
+ 6,
353
+ 10
354
+ ],
355
+ "7dim": [
356
+ 0,
357
+ 3,
358
+ 6,
359
+ 9
360
+ ],
361
+ "7aug": [
362
+ 0,
363
+ 4,
364
+ 8,
365
+ 11
366
+ ],
367
+ "9maj": [
368
+ 0,
369
+ 4,
370
+ 7,
371
+ 10,
372
+ 14
373
+ ],
374
+ "9min": [
375
+ 0,
376
+ 4,
377
+ 7,
378
+ 10,
379
+ 13
380
+ ]
381
+ },
382
+ "chord_tokens_with_root_note": false,
383
+ "chord_unknown": null,
384
+ "num_tempos": 32,
385
+ "tempo_range": [
386
+ 40,
387
+ 250
388
+ ],
389
+ "log_tempos": false,
390
+ "delete_equal_successive_tempo_changes": true,
391
+ "time_signature_range": {
392
+ "8": [
393
+ 3,
394
+ 12,
395
+ 6
396
+ ],
397
+ "4": [
398
+ 5,
399
+ 6,
400
+ 3,
401
+ 2,
402
+ 1,
403
+ 4
404
+ ]
405
+ },
406
+ "delete_equal_successive_time_sig_changes": true,
407
+ "sustain_pedal_duration": false,
408
+ "pitch_bend_range": [
409
+ -8192,
410
+ 8191,
411
+ 32
412
+ ],
413
+ "max_pitch_interval": 16,
414
+ "pitch_intervals_max_time_dist": 1,
415
+ "drums_pitch_range": [
416
+ 27,
417
+ 88
418
+ ],
419
+ "ac_polyphony_track": false,
420
+ "ac_polyphony_bar": false,
421
+ "ac_polyphony_min": 1,
422
+ "ac_polyphony_max": 6,
423
+ "ac_pitch_class_bar": false,
424
+ "ac_note_density_track": false,
425
+ "ac_note_density_track_min": 0,
426
+ "ac_note_density_track_max": 18,
427
+ "ac_note_density_bar": false,
428
+ "ac_note_density_bar_max": 18,
429
+ "ac_note_duration_bar": false,
430
+ "ac_note_duration_track": false,
431
+ "ac_repetition_track": false,
432
+ "ac_repetition_track_num_bins": 10,
433
+ "ac_repetition_track_num_consec_bars": 4,
434
+ "additional_params": {
435
+ "max_bar_embedding": null,
436
+ "use_bar_end_tokens": false,
437
+ "add_trailing_bars": false
438
+ }
439
+ },
440
+ "tokenization": "REMI",
441
+ "miditok_version": "3.0.6.post1",
442
+ "symusic_version": "0.5.9",
443
+ "hf_tokenizers_version": "0.20.3"
444
+ }
tokenizer_metadata.json ADDED
@@ -0,0 +1,167 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_type": "REMI",
3
+ "trained_bpe": false,
4
+ "tokenizer_params": {
5
+ "use_programs": true,
6
+ "use_velocities": false,
7
+ "one_token_stream_for_programs": true,
8
+ "use_chords": true,
9
+ "use_rests": false,
10
+ "use_tempos": true,
11
+ "use_time_signatures": true,
12
+ "use_sustain_pedals": false,
13
+ "use_pitch_bends": false,
14
+ "use_pitch_intervals": false,
15
+ "use_pitchdrum_tokens": true,
16
+ "remove_duplicated_notes": true,
17
+ "delete_equal_successive_tempo_changes": true,
18
+ "delete_equal_successive_time_sig_changes": true,
19
+ "sustain_pedal_duration": false,
20
+ "program_changes": true,
21
+ "special_tokens": [
22
+ "PAD",
23
+ "BOS",
24
+ "EOS",
25
+ "MASK"
26
+ ],
27
+ "use_note_duration_programs": [
28
+ -1,
29
+ 0,
30
+ 1,
31
+ 2,
32
+ 3,
33
+ 4,
34
+ 5,
35
+ 6,
36
+ 7,
37
+ 8,
38
+ 9,
39
+ 10,
40
+ 11,
41
+ 12,
42
+ 13,
43
+ 14,
44
+ 15,
45
+ 16,
46
+ 17,
47
+ 18,
48
+ 19,
49
+ 20,
50
+ 21,
51
+ 22,
52
+ 23,
53
+ 24,
54
+ 25,
55
+ 26,
56
+ 27,
57
+ 28,
58
+ 29,
59
+ 30,
60
+ 31,
61
+ 32,
62
+ 33,
63
+ 34,
64
+ 35,
65
+ 36,
66
+ 37,
67
+ 38,
68
+ 39,
69
+ 40,
70
+ 41,
71
+ 42,
72
+ 43,
73
+ 44,
74
+ 45,
75
+ 46,
76
+ 47,
77
+ 48,
78
+ 49,
79
+ 50,
80
+ 51,
81
+ 52,
82
+ 53,
83
+ 54,
84
+ 55,
85
+ 56,
86
+ 57,
87
+ 58,
88
+ 59,
89
+ 60,
90
+ 61,
91
+ 62,
92
+ 63,
93
+ 64,
94
+ 65,
95
+ 66,
96
+ 67,
97
+ 68,
98
+ 69,
99
+ 70,
100
+ 71,
101
+ 72,
102
+ 73,
103
+ 74,
104
+ 75,
105
+ 76,
106
+ 77,
107
+ 78,
108
+ 79,
109
+ 80,
110
+ 81,
111
+ 82,
112
+ 83,
113
+ 84,
114
+ 85,
115
+ 86,
116
+ 87,
117
+ 88,
118
+ 89,
119
+ 90,
120
+ 91,
121
+ 92,
122
+ 93,
123
+ 94,
124
+ 95,
125
+ 96,
126
+ 97,
127
+ 98,
128
+ 99,
129
+ 100,
130
+ 101,
131
+ 102,
132
+ 103,
133
+ 104,
134
+ 105,
135
+ 106,
136
+ 107,
137
+ 108,
138
+ 109,
139
+ 110,
140
+ 111,
141
+ 112,
142
+ 113,
143
+ 114,
144
+ 115,
145
+ 116,
146
+ 117,
147
+ 118,
148
+ 119,
149
+ 120,
150
+ 121,
151
+ 122,
152
+ 123,
153
+ 124,
154
+ 125,
155
+ 126,
156
+ 127
157
+ ],
158
+ "beat_res": {
159
+ "(0, 4)": 8,
160
+ "(4, 12)": 4
161
+ },
162
+ "pitch_range": [
163
+ 21,
164
+ 109
165
+ ]
166
+ }
167
+ }