Hugging Face's logo

voidful
/
bart-base-unit

Feature Extraction
Transformers
PyTorch
Safetensors
bart
Model card Files
xet
Community
1
voidful commited on
Commit
a94056b
·
1 Parent(s): bc36ccb

Upload tokenizer

Browse files
Files changed (2) hide show
  1. added_tokens.json +48 -0
  2. tokenizer.json +432 -0
added_tokens.json CHANGED
@@ -1114,10 +1114,58 @@
1114
  "v_tok_2": 50267,
1115
  "v_tok_20": 50285,
1116
  "v_tok_200": 50465,
 
 
 
 
 
 
 
 
 
 
1117
  "v_tok_201": 50466,
 
 
 
 
 
 
 
 
 
 
1118
  "v_tok_202": 50467,
 
 
 
 
 
 
 
 
 
 
1119
  "v_tok_203": 50468,
 
 
 
 
 
 
 
 
 
 
1120
  "v_tok_204": 50469,
 
 
 
 
 
 
 
 
1121
  "v_tok_205": 50470,
1122
  "v_tok_206": 50471,
1123
  "v_tok_207": 50472,
 
1114
  "v_tok_2": 50267,
1115
  "v_tok_20": 50285,
1116
  "v_tok_200": 50465,
1117
+ "v_tok_2000": 52265,
1118
+ "v_tok_2001": 52266,
1119
+ "v_tok_2002": 52267,
1120
+ "v_tok_2003": 52268,
1121
+ "v_tok_2004": 52269,
1122
+ "v_tok_2005": 52270,
1123
+ "v_tok_2006": 52271,
1124
+ "v_tok_2007": 52272,
1125
+ "v_tok_2008": 52273,
1126
+ "v_tok_2009": 52274,
1127
  "v_tok_201": 50466,
1128
+ "v_tok_2010": 52275,
1129
+ "v_tok_2011": 52276,
1130
+ "v_tok_2012": 52277,
1131
+ "v_tok_2013": 52278,
1132
+ "v_tok_2014": 52279,
1133
+ "v_tok_2015": 52280,
1134
+ "v_tok_2016": 52281,
1135
+ "v_tok_2017": 52282,
1136
+ "v_tok_2018": 52283,
1137
+ "v_tok_2019": 52284,
1138
  "v_tok_202": 50467,
1139
+ "v_tok_2020": 52285,
1140
+ "v_tok_2021": 52286,
1141
+ "v_tok_2022": 52287,
1142
+ "v_tok_2023": 52288,
1143
+ "v_tok_2024": 52289,
1144
+ "v_tok_2025": 52290,
1145
+ "v_tok_2026": 52291,
1146
+ "v_tok_2027": 52292,
1147
+ "v_tok_2028": 52293,
1148
+ "v_tok_2029": 52294,
1149
  "v_tok_203": 50468,
1150
+ "v_tok_2030": 52295,
1151
+ "v_tok_2031": 52296,
1152
+ "v_tok_2032": 52297,
1153
+ "v_tok_2033": 52298,
1154
+ "v_tok_2034": 52299,
1155
+ "v_tok_2035": 52300,
1156
+ "v_tok_2036": 52301,
1157
+ "v_tok_2037": 52302,
1158
+ "v_tok_2038": 52303,
1159
+ "v_tok_2039": 52304,
1160
  "v_tok_204": 50469,
1161
+ "v_tok_2040": 52305,
1162
+ "v_tok_2041": 52306,
1163
+ "v_tok_2042": 52307,
1164
+ "v_tok_2043": 52308,
1165
+ "v_tok_2044": 52309,
1166
+ "v_tok_2045": 52310,
1167
+ "v_tok_2046": 52311,
1168
+ "v_tok_2047": 52312,
1169
  "v_tok_205": 50470,
1170
  "v_tok_206": 50471,
1171
  "v_tok_207": 50472,
tokenizer.json CHANGED
@@ -18047,6 +18047,438 @@
18047
  "rstrip": false,
18048
  "normalized": true,
18049
  "special": false
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18050
  }
18051
  ],
18052
  "normalizer": null,
 
18047
  "rstrip": false,
18048
  "normalized": true,
18049
  "special": false
18050
+ },
18051
+ {
18052
+ "id": 52265,
18053
+ "content": "v_tok_2000",
18054
+ "single_word": false,
18055
+ "lstrip": false,
18056
+ "rstrip": false,
18057
+ "normalized": true,
18058
+ "special": false
18059
+ },
18060
+ {
18061
+ "id": 52266,
18062
+ "content": "v_tok_2001",
18063
+ "single_word": false,
18064
+ "lstrip": false,
18065
+ "rstrip": false,
18066
+ "normalized": true,
18067
+ "special": false
18068
+ },
18069
+ {
18070
+ "id": 52267,
18071
+ "content": "v_tok_2002",
18072
+ "single_word": false,
18073
+ "lstrip": false,
18074
+ "rstrip": false,
18075
+ "normalized": true,
18076
+ "special": false
18077
+ },
18078
+ {
18079
+ "id": 52268,
18080
+ "content": "v_tok_2003",
18081
+ "single_word": false,
18082
+ "lstrip": false,
18083
+ "rstrip": false,
18084
+ "normalized": true,
18085
+ "special": false
18086
+ },
18087
+ {
18088
+ "id": 52269,
18089
+ "content": "v_tok_2004",
18090
+ "single_word": false,
18091
+ "lstrip": false,
18092
+ "rstrip": false,
18093
+ "normalized": true,
18094
+ "special": false
18095
+ },
18096
+ {
18097
+ "id": 52270,
18098
+ "content": "v_tok_2005",
18099
+ "single_word": false,
18100
+ "lstrip": false,
18101
+ "rstrip": false,
18102
+ "normalized": true,
18103
+ "special": false
18104
+ },
18105
+ {
18106
+ "id": 52271,
18107
+ "content": "v_tok_2006",
18108
+ "single_word": false,
18109
+ "lstrip": false,
18110
+ "rstrip": false,
18111
+ "normalized": true,
18112
+ "special": false
18113
+ },
18114
+ {
18115
+ "id": 52272,
18116
+ "content": "v_tok_2007",
18117
+ "single_word": false,
18118
+ "lstrip": false,
18119
+ "rstrip": false,
18120
+ "normalized": true,
18121
+ "special": false
18122
+ },
18123
+ {
18124
+ "id": 52273,
18125
+ "content": "v_tok_2008",
18126
+ "single_word": false,
18127
+ "lstrip": false,
18128
+ "rstrip": false,
18129
+ "normalized": true,
18130
+ "special": false
18131
+ },
18132
+ {
18133
+ "id": 52274,
18134
+ "content": "v_tok_2009",
18135
+ "single_word": false,
18136
+ "lstrip": false,
18137
+ "rstrip": false,
18138
+ "normalized": true,
18139
+ "special": false
18140
+ },
18141
+ {
18142
+ "id": 52275,
18143
+ "content": "v_tok_2010",
18144
+ "single_word": false,
18145
+ "lstrip": false,
18146
+ "rstrip": false,
18147
+ "normalized": true,
18148
+ "special": false
18149
+ },
18150
+ {
18151
+ "id": 52276,
18152
+ "content": "v_tok_2011",
18153
+ "single_word": false,
18154
+ "lstrip": false,
18155
+ "rstrip": false,
18156
+ "normalized": true,
18157
+ "special": false
18158
+ },
18159
+ {
18160
+ "id": 52277,
18161
+ "content": "v_tok_2012",
18162
+ "single_word": false,
18163
+ "lstrip": false,
18164
+ "rstrip": false,
18165
+ "normalized": true,
18166
+ "special": false
18167
+ },
18168
+ {
18169
+ "id": 52278,
18170
+ "content": "v_tok_2013",
18171
+ "single_word": false,
18172
+ "lstrip": false,
18173
+ "rstrip": false,
18174
+ "normalized": true,
18175
+ "special": false
18176
+ },
18177
+ {
18178
+ "id": 52279,
18179
+ "content": "v_tok_2014",
18180
+ "single_word": false,
18181
+ "lstrip": false,
18182
+ "rstrip": false,
18183
+ "normalized": true,
18184
+ "special": false
18185
+ },
18186
+ {
18187
+ "id": 52280,
18188
+ "content": "v_tok_2015",
18189
+ "single_word": false,
18190
+ "lstrip": false,
18191
+ "rstrip": false,
18192
+ "normalized": true,
18193
+ "special": false
18194
+ },
18195
+ {
18196
+ "id": 52281,
18197
+ "content": "v_tok_2016",
18198
+ "single_word": false,
18199
+ "lstrip": false,
18200
+ "rstrip": false,
18201
+ "normalized": true,
18202
+ "special": false
18203
+ },
18204
+ {
18205
+ "id": 52282,
18206
+ "content": "v_tok_2017",
18207
+ "single_word": false,
18208
+ "lstrip": false,
18209
+ "rstrip": false,
18210
+ "normalized": true,
18211
+ "special": false
18212
+ },
18213
+ {
18214
+ "id": 52283,
18215
+ "content": "v_tok_2018",
18216
+ "single_word": false,
18217
+ "lstrip": false,
18218
+ "rstrip": false,
18219
+ "normalized": true,
18220
+ "special": false
18221
+ },
18222
+ {
18223
+ "id": 52284,
18224
+ "content": "v_tok_2019",
18225
+ "single_word": false,
18226
+ "lstrip": false,
18227
+ "rstrip": false,
18228
+ "normalized": true,
18229
+ "special": false
18230
+ },
18231
+ {
18232
+ "id": 52285,
18233
+ "content": "v_tok_2020",
18234
+ "single_word": false,
18235
+ "lstrip": false,
18236
+ "rstrip": false,
18237
+ "normalized": true,
18238
+ "special": false
18239
+ },
18240
+ {
18241
+ "id": 52286,
18242
+ "content": "v_tok_2021",
18243
+ "single_word": false,
18244
+ "lstrip": false,
18245
+ "rstrip": false,
18246
+ "normalized": true,
18247
+ "special": false
18248
+ },
18249
+ {
18250
+ "id": 52287,
18251
+ "content": "v_tok_2022",
18252
+ "single_word": false,
18253
+ "lstrip": false,
18254
+ "rstrip": false,
18255
+ "normalized": true,
18256
+ "special": false
18257
+ },
18258
+ {
18259
+ "id": 52288,
18260
+ "content": "v_tok_2023",
18261
+ "single_word": false,
18262
+ "lstrip": false,
18263
+ "rstrip": false,
18264
+ "normalized": true,
18265
+ "special": false
18266
+ },
18267
+ {
18268
+ "id": 52289,
18269
+ "content": "v_tok_2024",
18270
+ "single_word": false,
18271
+ "lstrip": false,
18272
+ "rstrip": false,
18273
+ "normalized": true,
18274
+ "special": false
18275
+ },
18276
+ {
18277
+ "id": 52290,
18278
+ "content": "v_tok_2025",
18279
+ "single_word": false,
18280
+ "lstrip": false,
18281
+ "rstrip": false,
18282
+ "normalized": true,
18283
+ "special": false
18284
+ },
18285
+ {
18286
+ "id": 52291,
18287
+ "content": "v_tok_2026",
18288
+ "single_word": false,
18289
+ "lstrip": false,
18290
+ "rstrip": false,
18291
+ "normalized": true,
18292
+ "special": false
18293
+ },
18294
+ {
18295
+ "id": 52292,
18296
+ "content": "v_tok_2027",
18297
+ "single_word": false,
18298
+ "lstrip": false,
18299
+ "rstrip": false,
18300
+ "normalized": true,
18301
+ "special": false
18302
+ },
18303
+ {
18304
+ "id": 52293,
18305
+ "content": "v_tok_2028",
18306
+ "single_word": false,
18307
+ "lstrip": false,
18308
+ "rstrip": false,
18309
+ "normalized": true,
18310
+ "special": false
18311
+ },
18312
+ {
18313
+ "id": 52294,
18314
+ "content": "v_tok_2029",
18315
+ "single_word": false,
18316
+ "lstrip": false,
18317
+ "rstrip": false,
18318
+ "normalized": true,
18319
+ "special": false
18320
+ },
18321
+ {
18322
+ "id": 52295,
18323
+ "content": "v_tok_2030",
18324
+ "single_word": false,
18325
+ "lstrip": false,
18326
+ "rstrip": false,
18327
+ "normalized": true,
18328
+ "special": false
18329
+ },
18330
+ {
18331
+ "id": 52296,
18332
+ "content": "v_tok_2031",
18333
+ "single_word": false,
18334
+ "lstrip": false,
18335
+ "rstrip": false,
18336
+ "normalized": true,
18337
+ "special": false
18338
+ },
18339
+ {
18340
+ "id": 52297,
18341
+ "content": "v_tok_2032",
18342
+ "single_word": false,
18343
+ "lstrip": false,
18344
+ "rstrip": false,
18345
+ "normalized": true,
18346
+ "special": false
18347
+ },
18348
+ {
18349
+ "id": 52298,
18350
+ "content": "v_tok_2033",
18351
+ "single_word": false,
18352
+ "lstrip": false,
18353
+ "rstrip": false,
18354
+ "normalized": true,
18355
+ "special": false
18356
+ },
18357
+ {
18358
+ "id": 52299,
18359
+ "content": "v_tok_2034",
18360
+ "single_word": false,
18361
+ "lstrip": false,
18362
+ "rstrip": false,
18363
+ "normalized": true,
18364
+ "special": false
18365
+ },
18366
+ {
18367
+ "id": 52300,
18368
+ "content": "v_tok_2035",
18369
+ "single_word": false,
18370
+ "lstrip": false,
18371
+ "rstrip": false,
18372
+ "normalized": true,
18373
+ "special": false
18374
+ },
18375
+ {
18376
+ "id": 52301,
18377
+ "content": "v_tok_2036",
18378
+ "single_word": false,
18379
+ "lstrip": false,
18380
+ "rstrip": false,
18381
+ "normalized": true,
18382
+ "special": false
18383
+ },
18384
+ {
18385
+ "id": 52302,
18386
+ "content": "v_tok_2037",
18387
+ "single_word": false,
18388
+ "lstrip": false,
18389
+ "rstrip": false,
18390
+ "normalized": true,
18391
+ "special": false
18392
+ },
18393
+ {
18394
+ "id": 52303,
18395
+ "content": "v_tok_2038",
18396
+ "single_word": false,
18397
+ "lstrip": false,
18398
+ "rstrip": false,
18399
+ "normalized": true,
18400
+ "special": false
18401
+ },
18402
+ {
18403
+ "id": 52304,
18404
+ "content": "v_tok_2039",
18405
+ "single_word": false,
18406
+ "lstrip": false,
18407
+ "rstrip": false,
18408
+ "normalized": true,
18409
+ "special": false
18410
+ },
18411
+ {
18412
+ "id": 52305,
18413
+ "content": "v_tok_2040",
18414
+ "single_word": false,
18415
+ "lstrip": false,
18416
+ "rstrip": false,
18417
+ "normalized": true,
18418
+ "special": false
18419
+ },
18420
+ {
18421
+ "id": 52306,
18422
+ "content": "v_tok_2041",
18423
+ "single_word": false,
18424
+ "lstrip": false,
18425
+ "rstrip": false,
18426
+ "normalized": true,
18427
+ "special": false
18428
+ },
18429
+ {
18430
+ "id": 52307,
18431
+ "content": "v_tok_2042",
18432
+ "single_word": false,
18433
+ "lstrip": false,
18434
+ "rstrip": false,
18435
+ "normalized": true,
18436
+ "special": false
18437
+ },
18438
+ {
18439
+ "id": 52308,
18440
+ "content": "v_tok_2043",
18441
+ "single_word": false,
18442
+ "lstrip": false,
18443
+ "rstrip": false,
18444
+ "normalized": true,
18445
+ "special": false
18446
+ },
18447
+ {
18448
+ "id": 52309,
18449
+ "content": "v_tok_2044",
18450
+ "single_word": false,
18451
+ "lstrip": false,
18452
+ "rstrip": false,
18453
+ "normalized": true,
18454
+ "special": false
18455
+ },
18456
+ {
18457
+ "id": 52310,
18458
+ "content": "v_tok_2045",
18459
+ "single_word": false,
18460
+ "lstrip": false,
18461
+ "rstrip": false,
18462
+ "normalized": true,
18463
+ "special": false
18464
+ },
18465
+ {
18466
+ "id": 52311,
18467
+ "content": "v_tok_2046",
18468
+ "single_word": false,
18469
+ "lstrip": false,
18470
+ "rstrip": false,
18471
+ "normalized": true,
18472
+ "special": false
18473
+ },
18474
+ {
18475
+ "id": 52312,
18476
+ "content": "v_tok_2047",
18477
+ "single_word": false,
18478
+ "lstrip": false,
18479
+ "rstrip": false,
18480
+ "normalized": true,
18481
+ "special": false
18482
  }
18483
  ],
18484
  "normalizer": null,