Hugging Face's logo

ayusha1103
/
singlemodel

Transformers
Model card Files
xet
Community
ayusha1103 commited on
Commit
92c0e31
·
verified ·
1 Parent(s): 3e231b3

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer.json +135 -81
tokenizer.json CHANGED
@@ -51,7 +51,7 @@
51
  ],
52
  "normalizer": null,
53
  "pre_tokenizer": {
54
- "type": "Whitespace"
55
  },
56
  "post_processor": null,
57
  "decoder": null,
@@ -63,86 +63,140 @@
63
  "[SEP]": 2,
64
  "[PAD]": 3,
65
  "[MASK]": 4,
66
- "=": 5,
67
- "NOTE_OFF": 6,
68
- "NOTE_ON": 7,
69
- "58": 8,
70
- "74": 9,
71
- "65": 10,
72
- "102": 11,
73
- "60": 12,
74
- "103": 13,
75
- "46": 14,
76
- "70": 15,
77
- "TIME_DELTA": 16,
78
- "92": 17,
79
- "80": 18,
80
- "98": 19,
81
- "77": 20,
82
- "1": 21,
83
- "BAR_END": 22,
84
- "BAR_START": 23,
85
- "90": 24,
86
- "79": 25,
87
- "91": 26,
88
- "81": 27,
89
- "93": 28,
90
- "72": 29,
91
- "82": 30,
92
- "61": 31,
93
- "86": 32,
94
- "62": 33,
95
- "95": 34,
96
- "101": 35,
97
- "73": 36,
98
- "78": 37,
99
- "76": 38,
100
- "100": 39,
101
- "67": 40,
102
- "84": 41,
103
- "64": 42,
104
- "68": 43,
105
- "83": 44,
106
- "87": 45,
107
- "63": 46,
108
- "88": 47,
109
- "94": 48,
110
- "89": 49,
111
- "0": 50,
112
- "37": 51,
113
- "57": 52,
114
- "85": 53,
115
- "96": 54,
116
- "36": 55,
117
- "59": 56,
118
- "66": 57,
119
- "DENSITY": 58,
120
- "INST": 59,
121
- "TRACK_END": 60,
122
- "TRACK_START": 61,
123
- "2": 62,
124
- "38": 63,
125
- "69": 64,
126
- "71": 65,
127
- "75": 66,
128
- "3": 67,
129
- "32": 68,
130
- "35": 69,
131
- "45": 70,
132
- "PIECE_END": 71,
133
- "PIECE_START": 72,
134
- "4": 73,
135
- "22": 74,
136
- "33": 75,
137
- "34": 76,
138
- "41": 77,
139
- "42": 78,
140
- "5": 79,
141
- "55": 80,
142
- "7": 81,
143
- "99": 82,
144
- "18": 83,
145
- "21": 84
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
146
  },
147
  "unk_token": "[UNK]"
148
  }
 
51
  ],
52
  "normalizer": null,
53
  "pre_tokenizer": {
54
+ "type": "WhitespaceSplit"
55
  },
56
  "post_processor": null,
57
  "decoder": null,
 
63
  "[SEP]": 2,
64
  "[PAD]": 3,
65
  "[MASK]": 4,
66
+ "NOTE_OFF=58": 5,
67
+ "NOTE_ON=58": 6,
68
+ "TIME_DELTA=1": 7,
69
+ "BAR_END": 8,
70
+ "BAR_START": 9,
71
+ "NOTE_OFF=74": 10,
72
+ "NOTE_ON=74": 11,
73
+ "NOTE_OFF=65": 12,
74
+ "NOTE_ON=65": 13,
75
+ "NOTE_OFF=102": 14,
76
+ "NOTE_ON=102": 15,
77
+ "NOTE_OFF=60": 16,
78
+ "NOTE_ON=60": 17,
79
+ "NOTE_OFF=103": 18,
80
+ "NOTE_OFF=46": 19,
81
+ "NOTE_ON=103": 20,
82
+ "NOTE_ON=46": 21,
83
+ "NOTE_OFF=70": 22,
84
+ "NOTE_ON=70": 23,
85
+ "NOTE_OFF=92": 24,
86
+ "NOTE_ON=92": 25,
87
+ "NOTE_OFF=80": 26,
88
+ "NOTE_OFF=98": 27,
89
+ "NOTE_ON=80": 28,
90
+ "NOTE_ON=98": 29,
91
+ "NOTE_OFF=77": 30,
92
+ "NOTE_ON=77": 31,
93
+ "NOTE_OFF=90": 32,
94
+ "NOTE_ON=90": 33,
95
+ "NOTE_OFF=79": 34,
96
+ "NOTE_OFF=91": 35,
97
+ "NOTE_ON=79": 36,
98
+ "NOTE_ON=91": 37,
99
+ "NOTE_OFF=81": 38,
100
+ "NOTE_ON=81": 39,
101
+ "NOTE_OFF=93": 40,
102
+ "NOTE_ON=93": 41,
103
+ "NOTE_OFF=72": 42,
104
+ "NOTE_OFF=82": 43,
105
+ "NOTE_ON=72": 44,
106
+ "NOTE_ON=82": 45,
107
+ "NOTE_OFF=61": 46,
108
+ "NOTE_ON=61": 47,
109
+ "NOTE_OFF=86": 48,
110
+ "NOTE_ON=86": 49,
111
+ "NOTE_OFF=62": 50,
112
+ "NOTE_OFF=95": 51,
113
+ "NOTE_ON=62": 52,
114
+ "NOTE_ON=95": 53,
115
+ "NOTE_OFF=101": 54,
116
+ "NOTE_OFF=73": 55,
117
+ "NOTE_ON=101": 56,
118
+ "NOTE_ON=73": 57,
119
+ "NOTE_OFF=78": 58,
120
+ "NOTE_ON=78": 59,
121
+ "NOTE_OFF=76": 60,
122
+ "NOTE_ON=76": 61,
123
+ "NOTE_OFF=100": 62,
124
+ "NOTE_OFF=67": 63,
125
+ "NOTE_OFF=84": 64,
126
+ "NOTE_ON=100": 65,
127
+ "NOTE_ON=67": 66,
128
+ "NOTE_ON=84": 67,
129
+ "NOTE_OFF=64": 68,
130
+ "NOTE_OFF=68": 69,
131
+ "NOTE_OFF=83": 70,
132
+ "NOTE_OFF=87": 71,
133
+ "NOTE_ON=64": 72,
134
+ "NOTE_ON=68": 73,
135
+ "NOTE_ON=83": 74,
136
+ "NOTE_ON=87": 75,
137
+ "NOTE_OFF=63": 76,
138
+ "NOTE_OFF=88": 77,
139
+ "NOTE_OFF=94": 78,
140
+ "NOTE_ON=63": 79,
141
+ "NOTE_ON=88": 80,
142
+ "NOTE_ON=94": 81,
143
+ "INST=0": 82,
144
+ "TRACK_END": 83,
145
+ "TRACK_START": 84,
146
+ "NOTE_OFF=89": 85,
147
+ "NOTE_ON=89": 86,
148
+ "NOTE_OFF=37": 87,
149
+ "NOTE_OFF=57": 88,
150
+ "NOTE_OFF=85": 89,
151
+ "NOTE_OFF=96": 90,
152
+ "NOTE_ON=37": 91,
153
+ "NOTE_ON=57": 92,
154
+ "NOTE_ON=85": 93,
155
+ "NOTE_ON=96": 94,
156
+ "TIME_DELTA=2": 95,
157
+ "DENSITY=0": 96,
158
+ "NOTE_OFF=36": 97,
159
+ "NOTE_OFF=59": 98,
160
+ "NOTE_OFF=66": 99,
161
+ "NOTE_ON=36": 100,
162
+ "NOTE_ON=59": 101,
163
+ "NOTE_ON=66": 102,
164
+ "PIECE_END": 103,
165
+ "PIECE_START": 104,
166
+ "TIME_DELTA=3": 105,
167
+ "NOTE_OFF=38": 106,
168
+ "NOTE_OFF=69": 107,
169
+ "NOTE_OFF=71": 108,
170
+ "NOTE_OFF=75": 109,
171
+ "NOTE_ON=38": 110,
172
+ "NOTE_ON=69": 111,
173
+ "NOTE_ON=71": 112,
174
+ "NOTE_ON=75": 113,
175
+ "TIME_DELTA=4": 114,
176
+ "NOTE_OFF=32": 115,
177
+ "NOTE_OFF=35": 116,
178
+ "NOTE_OFF=45": 117,
179
+ "NOTE_ON=32": 118,
180
+ "NOTE_ON=35": 119,
181
+ "NOTE_ON=45": 120,
182
+ "TIME_DELTA=5": 121,
183
+ "TIME_DELTA=7": 122,
184
+ "DENSITY=18": 123,
185
+ "DENSITY=21": 124,
186
+ "DENSITY=22": 125,
187
+ "NOTE_OFF=33": 126,
188
+ "NOTE_OFF=34": 127,
189
+ "NOTE_OFF=41": 128,
190
+ "NOTE_OFF=42": 129,
191
+ "NOTE_OFF=55": 130,
192
+ "NOTE_OFF=99": 131,
193
+ "NOTE_ON=33": 132,
194
+ "NOTE_ON=34": 133,
195
+ "NOTE_ON=41": 134,
196
+ "NOTE_ON=42": 135,
197
+ "NOTE_ON=55": 136,
198
+ "NOTE_ON=99": 137,
199
+ "TIME_DELTA=22": 138
200
  },
201
  "unk_token": "[UNK]"
202
  }