ilyyeees commited on
Commit
c5c0794
·
verified ·
1 Parent(s): 5358ab7

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -127
tokenizer_config.json CHANGED
@@ -1156,133 +1156,7 @@
1156
  "clean_up_tokenization_spaces": false,
1157
  "eos_token": "</s>",
1158
  "extra_ids": 0,
1159
- "extra_special_tokens": [
1160
- "<extra_id_0>",
1161
- "<extra_id_1>",
1162
- "<extra_id_2>",
1163
- "<extra_id_3>",
1164
- "<extra_id_4>",
1165
- "<extra_id_5>",
1166
- "<extra_id_6>",
1167
- "<extra_id_7>",
1168
- "<extra_id_8>",
1169
- "<extra_id_9>",
1170
- "<extra_id_10>",
1171
- "<extra_id_11>",
1172
- "<extra_id_12>",
1173
- "<extra_id_13>",
1174
- "<extra_id_14>",
1175
- "<extra_id_15>",
1176
- "<extra_id_16>",
1177
- "<extra_id_17>",
1178
- "<extra_id_18>",
1179
- "<extra_id_19>",
1180
- "<extra_id_20>",
1181
- "<extra_id_21>",
1182
- "<extra_id_22>",
1183
- "<extra_id_23>",
1184
- "<extra_id_24>",
1185
- "<extra_id_25>",
1186
- "<extra_id_26>",
1187
- "<extra_id_27>",
1188
- "<extra_id_28>",
1189
- "<extra_id_29>",
1190
- "<extra_id_30>",
1191
- "<extra_id_31>",
1192
- "<extra_id_32>",
1193
- "<extra_id_33>",
1194
- "<extra_id_34>",
1195
- "<extra_id_35>",
1196
- "<extra_id_36>",
1197
- "<extra_id_37>",
1198
- "<extra_id_38>",
1199
- "<extra_id_39>",
1200
- "<extra_id_40>",
1201
- "<extra_id_41>",
1202
- "<extra_id_42>",
1203
- "<extra_id_43>",
1204
- "<extra_id_44>",
1205
- "<extra_id_45>",
1206
- "<extra_id_46>",
1207
- "<extra_id_47>",
1208
- "<extra_id_48>",
1209
- "<extra_id_49>",
1210
- "<extra_id_50>",
1211
- "<extra_id_51>",
1212
- "<extra_id_52>",
1213
- "<extra_id_53>",
1214
- "<extra_id_54>",
1215
- "<extra_id_55>",
1216
- "<extra_id_56>",
1217
- "<extra_id_57>",
1218
- "<extra_id_58>",
1219
- "<extra_id_59>",
1220
- "<extra_id_60>",
1221
- "<extra_id_61>",
1222
- "<extra_id_62>",
1223
- "<extra_id_63>",
1224
- "<extra_id_64>",
1225
- "<extra_id_65>",
1226
- "<extra_id_66>",
1227
- "<extra_id_67>",
1228
- "<extra_id_68>",
1229
- "<extra_id_69>",
1230
- "<extra_id_70>",
1231
- "<extra_id_71>",
1232
- "<extra_id_72>",
1233
- "<extra_id_73>",
1234
- "<extra_id_74>",
1235
- "<extra_id_75>",
1236
- "<extra_id_76>",
1237
- "<extra_id_77>",
1238
- "<extra_id_78>",
1239
- "<extra_id_79>",
1240
- "<extra_id_80>",
1241
- "<extra_id_81>",
1242
- "<extra_id_82>",
1243
- "<extra_id_83>",
1244
- "<extra_id_84>",
1245
- "<extra_id_85>",
1246
- "<extra_id_86>",
1247
- "<extra_id_87>",
1248
- "<extra_id_88>",
1249
- "<extra_id_89>",
1250
- "<extra_id_90>",
1251
- "<extra_id_91>",
1252
- "<extra_id_92>",
1253
- "<extra_id_93>",
1254
- "<extra_id_94>",
1255
- "<extra_id_95>",
1256
- "<extra_id_96>",
1257
- "<extra_id_97>",
1258
- "<extra_id_98>",
1259
- "<extra_id_99>",
1260
- "<extra_id_100>",
1261
- "<extra_id_101>",
1262
- "<extra_id_102>",
1263
- "<extra_id_103>",
1264
- "<extra_id_104>",
1265
- "<extra_id_105>",
1266
- "<extra_id_106>",
1267
- "<extra_id_107>",
1268
- "<extra_id_108>",
1269
- "<extra_id_109>",
1270
- "<extra_id_110>",
1271
- "<extra_id_111>",
1272
- "<extra_id_112>",
1273
- "<extra_id_113>",
1274
- "<extra_id_114>",
1275
- "<extra_id_115>",
1276
- "<extra_id_116>",
1277
- "<extra_id_117>",
1278
- "<extra_id_118>",
1279
- "<extra_id_119>",
1280
- "<extra_id_120>",
1281
- "<extra_id_121>",
1282
- "<extra_id_122>",
1283
- "<extra_id_123>",
1284
- "<extra_id_124>"
1285
- ],
1286
  "is_local": true,
1287
  "model_max_length": 1000000000000000019884624838656,
1288
  "model_specific_special_tokens": {},
 
1156
  "clean_up_tokenization_spaces": false,
1157
  "eos_token": "</s>",
1158
  "extra_ids": 0,
1159
+ "extra_special_tokens": {},
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1160
  "is_local": true,
1161
  "model_max_length": 1000000000000000019884624838656,
1162
  "model_specific_special_tokens": {},