TylerHMC commited on
Commit
b115bc3
·
verified ·
1 Parent(s): e9b83f4

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. tokenizer.json +0 -0
  2. tokenizer_metadata.json +176 -0
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_metadata.json ADDED
@@ -0,0 +1,176 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_type": "REMI",
3
+ "trained_bpe": true,
4
+ "bpe_vocab_size": 8000,
5
+ "final_vocab_size": 8000,
6
+ "bpe_train_sample_size": 8000,
7
+ "bpe_sample_seed": 42,
8
+ "tokenizer_params": {
9
+ "use_programs": true,
10
+ "use_velocities": true,
11
+ "one_token_stream_for_programs": true,
12
+ "use_chords": true,
13
+ "use_rests": false,
14
+ "use_tempos": true,
15
+ "use_time_signatures": true,
16
+ "use_sustain_pedals": false,
17
+ "use_pitch_bends": false,
18
+ "use_pitch_intervals": false,
19
+ "use_pitchdrum_tokens": true,
20
+ "remove_duplicated_notes": true,
21
+ "delete_equal_successive_tempo_changes": true,
22
+ "delete_equal_successive_time_sig_changes": true,
23
+ "sustain_pedal_duration": false,
24
+ "program_changes": true,
25
+ "special_tokens": [
26
+ "PAD",
27
+ "BOS",
28
+ "EOS",
29
+ "MASK"
30
+ ],
31
+ "use_note_duration_programs": [
32
+ -1,
33
+ 0,
34
+ 1,
35
+ 2,
36
+ 3,
37
+ 4,
38
+ 5,
39
+ 6,
40
+ 7,
41
+ 8,
42
+ 9,
43
+ 10,
44
+ 11,
45
+ 12,
46
+ 13,
47
+ 14,
48
+ 15,
49
+ 16,
50
+ 17,
51
+ 18,
52
+ 19,
53
+ 20,
54
+ 21,
55
+ 22,
56
+ 23,
57
+ 24,
58
+ 25,
59
+ 26,
60
+ 27,
61
+ 28,
62
+ 29,
63
+ 30,
64
+ 31,
65
+ 32,
66
+ 33,
67
+ 34,
68
+ 35,
69
+ 36,
70
+ 37,
71
+ 38,
72
+ 39,
73
+ 40,
74
+ 41,
75
+ 42,
76
+ 43,
77
+ 44,
78
+ 45,
79
+ 46,
80
+ 47,
81
+ 48,
82
+ 49,
83
+ 50,
84
+ 51,
85
+ 52,
86
+ 53,
87
+ 54,
88
+ 55,
89
+ 56,
90
+ 57,
91
+ 58,
92
+ 59,
93
+ 60,
94
+ 61,
95
+ 62,
96
+ 63,
97
+ 64,
98
+ 65,
99
+ 66,
100
+ 67,
101
+ 68,
102
+ 69,
103
+ 70,
104
+ 71,
105
+ 72,
106
+ 73,
107
+ 74,
108
+ 75,
109
+ 76,
110
+ 77,
111
+ 78,
112
+ 79,
113
+ 80,
114
+ 81,
115
+ 82,
116
+ 83,
117
+ 84,
118
+ 85,
119
+ 86,
120
+ 87,
121
+ 88,
122
+ 89,
123
+ 90,
124
+ 91,
125
+ 92,
126
+ 93,
127
+ 94,
128
+ 95,
129
+ 96,
130
+ 97,
131
+ 98,
132
+ 99,
133
+ 100,
134
+ 101,
135
+ 102,
136
+ 103,
137
+ 104,
138
+ 105,
139
+ 106,
140
+ 107,
141
+ 108,
142
+ 109,
143
+ 110,
144
+ 111,
145
+ 112,
146
+ 113,
147
+ 114,
148
+ 115,
149
+ 116,
150
+ 117,
151
+ 118,
152
+ 119,
153
+ 120,
154
+ 121,
155
+ 122,
156
+ 123,
157
+ 124,
158
+ 125,
159
+ 126,
160
+ 127
161
+ ],
162
+ "beat_res": {
163
+ "(0, 4)": 8,
164
+ "(4, 12)": 4
165
+ },
166
+ "pitch_range": [
167
+ 21,
168
+ 109
169
+ ],
170
+ "num_velocities": 32
171
+ },
172
+ "chunk_size": 2048,
173
+ "min_chunk_token_ratio": 0.5,
174
+ "min_chunk_tokens": 1024,
175
+ "padding": false
176
+ }