Upload tokenizer
Browse files- tokenizer_config.json +0 -240
tokenizer_config.json
CHANGED
|
@@ -1114,246 +1114,6 @@
|
|
| 1114 |
"single_word": false,
|
| 1115 |
"special": false
|
| 1116 |
},
|
| 1117 |
-
"139": {
|
| 1118 |
-
"content": "▁▁",
|
| 1119 |
-
"lstrip": false,
|
| 1120 |
-
"normalized": false,
|
| 1121 |
-
"rstrip": false,
|
| 1122 |
-
"single_word": false,
|
| 1123 |
-
"special": false
|
| 1124 |
-
},
|
| 1125 |
-
"140": {
|
| 1126 |
-
"content": "▁▁▁",
|
| 1127 |
-
"lstrip": false,
|
| 1128 |
-
"normalized": false,
|
| 1129 |
-
"rstrip": false,
|
| 1130 |
-
"single_word": false,
|
| 1131 |
-
"special": false
|
| 1132 |
-
},
|
| 1133 |
-
"141": {
|
| 1134 |
-
"content": "▁▁▁▁",
|
| 1135 |
-
"lstrip": false,
|
| 1136 |
-
"normalized": false,
|
| 1137 |
-
"rstrip": false,
|
| 1138 |
-
"single_word": false,
|
| 1139 |
-
"special": false
|
| 1140 |
-
},
|
| 1141 |
-
"142": {
|
| 1142 |
-
"content": "▁▁▁▁▁",
|
| 1143 |
-
"lstrip": false,
|
| 1144 |
-
"normalized": false,
|
| 1145 |
-
"rstrip": false,
|
| 1146 |
-
"single_word": false,
|
| 1147 |
-
"special": false
|
| 1148 |
-
},
|
| 1149 |
-
"143": {
|
| 1150 |
-
"content": "▁▁▁▁▁▁",
|
| 1151 |
-
"lstrip": false,
|
| 1152 |
-
"normalized": false,
|
| 1153 |
-
"rstrip": false,
|
| 1154 |
-
"single_word": false,
|
| 1155 |
-
"special": false
|
| 1156 |
-
},
|
| 1157 |
-
"144": {
|
| 1158 |
-
"content": "▁▁▁▁▁▁▁",
|
| 1159 |
-
"lstrip": false,
|
| 1160 |
-
"normalized": false,
|
| 1161 |
-
"rstrip": false,
|
| 1162 |
-
"single_word": false,
|
| 1163 |
-
"special": false
|
| 1164 |
-
},
|
| 1165 |
-
"145": {
|
| 1166 |
-
"content": "▁▁▁▁▁▁▁▁",
|
| 1167 |
-
"lstrip": false,
|
| 1168 |
-
"normalized": false,
|
| 1169 |
-
"rstrip": false,
|
| 1170 |
-
"single_word": false,
|
| 1171 |
-
"special": false
|
| 1172 |
-
},
|
| 1173 |
-
"146": {
|
| 1174 |
-
"content": "▁▁▁▁▁▁▁▁▁",
|
| 1175 |
-
"lstrip": false,
|
| 1176 |
-
"normalized": false,
|
| 1177 |
-
"rstrip": false,
|
| 1178 |
-
"single_word": false,
|
| 1179 |
-
"special": false
|
| 1180 |
-
},
|
| 1181 |
-
"147": {
|
| 1182 |
-
"content": "▁▁▁▁▁▁▁▁▁▁",
|
| 1183 |
-
"lstrip": false,
|
| 1184 |
-
"normalized": false,
|
| 1185 |
-
"rstrip": false,
|
| 1186 |
-
"single_word": false,
|
| 1187 |
-
"special": false
|
| 1188 |
-
},
|
| 1189 |
-
"148": {
|
| 1190 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁",
|
| 1191 |
-
"lstrip": false,
|
| 1192 |
-
"normalized": false,
|
| 1193 |
-
"rstrip": false,
|
| 1194 |
-
"single_word": false,
|
| 1195 |
-
"special": false
|
| 1196 |
-
},
|
| 1197 |
-
"149": {
|
| 1198 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1199 |
-
"lstrip": false,
|
| 1200 |
-
"normalized": false,
|
| 1201 |
-
"rstrip": false,
|
| 1202 |
-
"single_word": false,
|
| 1203 |
-
"special": false
|
| 1204 |
-
},
|
| 1205 |
-
"150": {
|
| 1206 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1207 |
-
"lstrip": false,
|
| 1208 |
-
"normalized": false,
|
| 1209 |
-
"rstrip": false,
|
| 1210 |
-
"single_word": false,
|
| 1211 |
-
"special": false
|
| 1212 |
-
},
|
| 1213 |
-
"151": {
|
| 1214 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1215 |
-
"lstrip": false,
|
| 1216 |
-
"normalized": false,
|
| 1217 |
-
"rstrip": false,
|
| 1218 |
-
"single_word": false,
|
| 1219 |
-
"special": false
|
| 1220 |
-
},
|
| 1221 |
-
"152": {
|
| 1222 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1223 |
-
"lstrip": false,
|
| 1224 |
-
"normalized": false,
|
| 1225 |
-
"rstrip": false,
|
| 1226 |
-
"single_word": false,
|
| 1227 |
-
"special": false
|
| 1228 |
-
},
|
| 1229 |
-
"153": {
|
| 1230 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1231 |
-
"lstrip": false,
|
| 1232 |
-
"normalized": false,
|
| 1233 |
-
"rstrip": false,
|
| 1234 |
-
"single_word": false,
|
| 1235 |
-
"special": false
|
| 1236 |
-
},
|
| 1237 |
-
"154": {
|
| 1238 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1239 |
-
"lstrip": false,
|
| 1240 |
-
"normalized": false,
|
| 1241 |
-
"rstrip": false,
|
| 1242 |
-
"single_word": false,
|
| 1243 |
-
"special": false
|
| 1244 |
-
},
|
| 1245 |
-
"155": {
|
| 1246 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1247 |
-
"lstrip": false,
|
| 1248 |
-
"normalized": false,
|
| 1249 |
-
"rstrip": false,
|
| 1250 |
-
"single_word": false,
|
| 1251 |
-
"special": false
|
| 1252 |
-
},
|
| 1253 |
-
"156": {
|
| 1254 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1255 |
-
"lstrip": false,
|
| 1256 |
-
"normalized": false,
|
| 1257 |
-
"rstrip": false,
|
| 1258 |
-
"single_word": false,
|
| 1259 |
-
"special": false
|
| 1260 |
-
},
|
| 1261 |
-
"157": {
|
| 1262 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1263 |
-
"lstrip": false,
|
| 1264 |
-
"normalized": false,
|
| 1265 |
-
"rstrip": false,
|
| 1266 |
-
"single_word": false,
|
| 1267 |
-
"special": false
|
| 1268 |
-
},
|
| 1269 |
-
"158": {
|
| 1270 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1271 |
-
"lstrip": false,
|
| 1272 |
-
"normalized": false,
|
| 1273 |
-
"rstrip": false,
|
| 1274 |
-
"single_word": false,
|
| 1275 |
-
"special": false
|
| 1276 |
-
},
|
| 1277 |
-
"159": {
|
| 1278 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1279 |
-
"lstrip": false,
|
| 1280 |
-
"normalized": false,
|
| 1281 |
-
"rstrip": false,
|
| 1282 |
-
"single_word": false,
|
| 1283 |
-
"special": false
|
| 1284 |
-
},
|
| 1285 |
-
"160": {
|
| 1286 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1287 |
-
"lstrip": false,
|
| 1288 |
-
"normalized": false,
|
| 1289 |
-
"rstrip": false,
|
| 1290 |
-
"single_word": false,
|
| 1291 |
-
"special": false
|
| 1292 |
-
},
|
| 1293 |
-
"161": {
|
| 1294 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1295 |
-
"lstrip": false,
|
| 1296 |
-
"normalized": false,
|
| 1297 |
-
"rstrip": false,
|
| 1298 |
-
"single_word": false,
|
| 1299 |
-
"special": false
|
| 1300 |
-
},
|
| 1301 |
-
"162": {
|
| 1302 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1303 |
-
"lstrip": false,
|
| 1304 |
-
"normalized": false,
|
| 1305 |
-
"rstrip": false,
|
| 1306 |
-
"single_word": false,
|
| 1307 |
-
"special": false
|
| 1308 |
-
},
|
| 1309 |
-
"163": {
|
| 1310 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1311 |
-
"lstrip": false,
|
| 1312 |
-
"normalized": false,
|
| 1313 |
-
"rstrip": false,
|
| 1314 |
-
"single_word": false,
|
| 1315 |
-
"special": false
|
| 1316 |
-
},
|
| 1317 |
-
"164": {
|
| 1318 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1319 |
-
"lstrip": false,
|
| 1320 |
-
"normalized": false,
|
| 1321 |
-
"rstrip": false,
|
| 1322 |
-
"single_word": false,
|
| 1323 |
-
"special": false
|
| 1324 |
-
},
|
| 1325 |
-
"165": {
|
| 1326 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1327 |
-
"lstrip": false,
|
| 1328 |
-
"normalized": false,
|
| 1329 |
-
"rstrip": false,
|
| 1330 |
-
"single_word": false,
|
| 1331 |
-
"special": false
|
| 1332 |
-
},
|
| 1333 |
-
"166": {
|
| 1334 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1335 |
-
"lstrip": false,
|
| 1336 |
-
"normalized": false,
|
| 1337 |
-
"rstrip": false,
|
| 1338 |
-
"single_word": false,
|
| 1339 |
-
"special": false
|
| 1340 |
-
},
|
| 1341 |
-
"167": {
|
| 1342 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1343 |
-
"lstrip": false,
|
| 1344 |
-
"normalized": false,
|
| 1345 |
-
"rstrip": false,
|
| 1346 |
-
"single_word": false,
|
| 1347 |
-
"special": false
|
| 1348 |
-
},
|
| 1349 |
-
"168": {
|
| 1350 |
-
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
|
| 1351 |
-
"lstrip": false,
|
| 1352 |
-
"normalized": false,
|
| 1353 |
-
"rstrip": false,
|
| 1354 |
-
"single_word": false,
|
| 1355 |
-
"special": false
|
| 1356 |
-
},
|
| 1357 |
"169": {
|
| 1358 |
"content": "<table>",
|
| 1359 |
"lstrip": false,
|
|
|
|
| 1114 |
"single_word": false,
|
| 1115 |
"special": false
|
| 1116 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1117 |
"169": {
|
| 1118 |
"content": "<table>",
|
| 1119 |
"lstrip": false,
|