Safetensors
LauraWang1107 commited on
Commit
e54394f
·
verified ·
1 Parent(s): 4df1894

Delete tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +0 -700
tokenizer.json DELETED
@@ -1,700 +0,0 @@
1
- {
2
- "version": "1.0",
3
- "truncation": null,
4
- "padding": null,
5
- "added_tokens": [
6
- {
7
- "id": 0,
8
- "content": "[PAD]",
9
- "single_word": false,
10
- "lstrip": false,
11
- "rstrip": false,
12
- "normalized": false,
13
- "special": true
14
- },
15
- {
16
- "id": 11,
17
- "content": "[UNK]",
18
- "single_word": false,
19
- "lstrip": false,
20
- "rstrip": false,
21
- "normalized": false,
22
- "special": true
23
- },
24
- {
25
- "id": 12,
26
- "content": "[CLS]",
27
- "single_word": false,
28
- "lstrip": false,
29
- "rstrip": false,
30
- "normalized": false,
31
- "special": true
32
- },
33
- {
34
- "id": 13,
35
- "content": "[SEP]",
36
- "single_word": false,
37
- "lstrip": false,
38
- "rstrip": false,
39
- "normalized": false,
40
- "special": true
41
- },
42
- {
43
- "id": 14,
44
- "content": "[MASK]",
45
- "single_word": false,
46
- "lstrip": true,
47
- "rstrip": false,
48
- "normalized": false,
49
- "special": true
50
- },
51
- {
52
- "id": 591,
53
- "content": "<s>",
54
- "single_word": false,
55
- "lstrip": false,
56
- "rstrip": false,
57
- "normalized": true,
58
- "special": true
59
- },
60
- {
61
- "id": 592,
62
- "content": "</s>",
63
- "single_word": false,
64
- "lstrip": false,
65
- "rstrip": false,
66
- "normalized": true,
67
- "special": true
68
- }
69
- ],
70
- "normalizer": null,
71
- "pre_tokenizer": {
72
- "type": "ByteLevel",
73
- "add_prefix_space": false,
74
- "trim_offsets": true,
75
- "use_regex": true
76
- },
77
- "post_processor": {
78
- "type": "RobertaProcessing",
79
- "sep": [
80
- "[SEP]",
81
- 13
82
- ],
83
- "cls": [
84
- "[CLS]",
85
- 12
86
- ],
87
- "trim_offsets": true,
88
- "add_prefix_space": false
89
- },
90
- "decoder": {
91
- "type": "ByteLevel",
92
- "add_prefix_space": true,
93
- "trim_offsets": true,
94
- "use_regex": true
95
- },
96
- "model": {
97
- "type": "BPE",
98
- "dropout": null,
99
- "unk_token": null,
100
- "continuing_subword_prefix": "",
101
- "end_of_word_suffix": "",
102
- "fuse_unk": false,
103
- "byte_fallback": false,
104
- "ignore_merges": false,
105
- "vocab": {
106
- "[PAD]": 0,
107
- "[unused1]": 1,
108
- "[unused2]": 2,
109
- "[unused3]": 3,
110
- "[unused4]": 4,
111
- "[unused5]": 5,
112
- "[unused6]": 6,
113
- "[unused7]": 7,
114
- "[unused8]": 8,
115
- "[unused9]": 9,
116
- "[unused10]": 10,
117
- "[UNK]": 11,
118
- "[CLS]": 12,
119
- "[SEP]": 13,
120
- "[MASK]": 14,
121
- "c": 15,
122
- "C": 16,
123
- "(": 17,
124
- ")": 18,
125
- "O": 19,
126
- "1": 20,
127
- "2": 21,
128
- "=": 22,
129
- "N": 23,
130
- ".": 24,
131
- "n": 25,
132
- "3": 26,
133
- "F": 27,
134
- "Cl": 28,
135
- ">>": 29,
136
- "~": 30,
137
- "-": 31,
138
- "4": 32,
139
- "[C@H]": 33,
140
- "S": 34,
141
- "[C@@H]": 35,
142
- "[O-]": 36,
143
- "Br": 37,
144
- "#": 38,
145
- "/": 39,
146
- "[nH]": 40,
147
- "[N+]": 41,
148
- "s": 42,
149
- "5": 43,
150
- "o": 44,
151
- "P": 45,
152
- "[Na+]": 46,
153
- "[Si]": 47,
154
- "I": 48,
155
- "[Na]": 49,
156
- "[Pd]": 50,
157
- "[K+]": 51,
158
- "[K]": 52,
159
- "[P]": 53,
160
- "B": 54,
161
- "[C@]": 55,
162
- "[C@@]": 56,
163
- "[Cl-]": 57,
164
- "6": 58,
165
- "[OH-]": 59,
166
- "\\": 60,
167
- "[N-]": 61,
168
- "[Li]": 62,
169
- "[H]": 63,
170
- "[2H]": 64,
171
- "[NH4+]": 65,
172
- "[c-]": 66,
173
- "[P-]": 67,
174
- "[Cs+]": 68,
175
- "[Li+]": 69,
176
- "[Cs]": 70,
177
- "[NaH]": 71,
178
- "[H-]": 72,
179
- "[O+]": 73,
180
- "[BH4-]": 74,
181
- "[Cu]": 75,
182
- "7": 76,
183
- "[Mg]": 77,
184
- "[Fe+2]": 78,
185
- "[n+]": 79,
186
- "[Sn]": 80,
187
- "[BH-]": 81,
188
- "[Pd+2]": 82,
189
- "[CH]": 83,
190
- "[I-]": 84,
191
- "[Br-]": 85,
192
- "[C-]": 86,
193
- "[Zn]": 87,
194
- "[B-]": 88,
195
- "[F-]": 89,
196
- "[Al]": 90,
197
- "[P+]": 91,
198
- "[BH3-]": 92,
199
- "[Fe]": 93,
200
- "[C]": 94,
201
- "[AlH4]": 95,
202
- "[Ni]": 96,
203
- "[SiH]": 97,
204
- "8": 98,
205
- "[Cu+2]": 99,
206
- "[Mn]": 100,
207
- "[AlH]": 101,
208
- "[nH+]": 102,
209
- "[AlH4-]": 103,
210
- "[O-2]": 104,
211
- "[Cr]": 105,
212
- "[Mg+2]": 106,
213
- "[NH3+]": 107,
214
- "[S@]": 108,
215
- "[Pt]": 109,
216
- "[Al+3]": 110,
217
- "[S@@]": 111,
218
- "[S-]": 112,
219
- "[Ti]": 113,
220
- "[Zn+2]": 114,
221
- "[PH]": 115,
222
- "[NH2+]": 116,
223
- "[Ru]": 117,
224
- "[Ag+]": 118,
225
- "[S+]": 119,
226
- "[I+3]": 120,
227
- "[NH+]": 121,
228
- "[Ca+2]": 122,
229
- "[Ag]": 123,
230
- "9": 124,
231
- "[Os]": 125,
232
- "[Se]": 126,
233
- "[SiH2]": 127,
234
- "[Ca]": 128,
235
- "[Ti+4]": 129,
236
- "[Ac]": 130,
237
- "[Cu+]": 131,
238
- "[S]": 132,
239
- "[Rh]": 133,
240
- "[Cl+3]": 134,
241
- "[cH-]": 135,
242
- "[Zn+]": 136,
243
- "[O]": 137,
244
- "[Cl+]": 138,
245
- "[SH]": 139,
246
- "[H+]": 140,
247
- "[Pd+]": 141,
248
- "[se]": 142,
249
- "[PH+]": 143,
250
- "[I]": 144,
251
- "[Pt+2]": 145,
252
- "[C+]": 146,
253
- "[Mg+]": 147,
254
- "[Hg]": 148,
255
- "[W]": 149,
256
- "[SnH]": 150,
257
- "[SiH3]": 151,
258
- "[Fe+3]": 152,
259
- "[NH]": 153,
260
- "[Mo]": 154,
261
- "[CH2+]": 155,
262
- "%10": 156,
263
- "[CH2-]": 157,
264
- "[CH2]": 158,
265
- "[n-]": 159,
266
- "[Ce+4]": 160,
267
- "[NH-]": 161,
268
- "[Co]": 162,
269
- "[I+]": 163,
270
- "[PH2]": 164,
271
- "[Pt+4]": 165,
272
- "[Ce]": 166,
273
- "[B]": 167,
274
- "[Sn+2]": 168,
275
- "[Ba+2]": 169,
276
- "%11": 170,
277
- "[Fe-3]": 171,
278
- "[18F]": 172,
279
- "[SH-]": 173,
280
- "[Pb+2]": 174,
281
- "[Os-2]": 175,
282
- "[Zr+4]": 176,
283
- "[N]": 177,
284
- "[Ir]": 178,
285
- "[Bi]": 179,
286
- "[Ni+2]": 180,
287
- "[P@]": 181,
288
- "[Co+2]": 182,
289
- "[s+]": 183,
290
- "[As]": 184,
291
- "[P+3]": 185,
292
- "[Hg+2]": 186,
293
- "[Yb+3]": 187,
294
- "[CH-]": 188,
295
- "[Zr+2]": 189,
296
- "[Mn+2]": 190,
297
- "[CH+]": 191,
298
- "[In]": 192,
299
- "[KH]": 193,
300
- "[Ce+3]": 194,
301
- "[Zr]": 195,
302
- "[AlH2-]": 196,
303
- "[OH2+]": 197,
304
- "[Ti+3]": 198,
305
- "[Rh+2]": 199,
306
- "[Sb]": 200,
307
- "[S-2]": 201,
308
- "%12": 202,
309
- "[P@@]": 203,
310
- "[Si@H]": 204,
311
- "[Mn+4]": 205,
312
- "p": 206,
313
- "[Ba]": 207,
314
- "[NH2-]": 208,
315
- "[Ge]": 209,
316
- "[Pb+4]": 210,
317
- "[Cr+3]": 211,
318
- "[Au]": 212,
319
- "[LiH]": 213,
320
- "[Sc+3]": 214,
321
- "[o+]": 215,
322
- "[Rh-3]": 216,
323
- "%13": 217,
324
- "[Br]": 218,
325
- "[Sb-]": 219,
326
- "[S@+]": 220,
327
- "[I+2]": 221,
328
- "[Ar]": 222,
329
- "[V]": 223,
330
- "[Cu-]": 224,
331
- "[Al-]": 225,
332
- "[Te]": 226,
333
- "[13c]": 227,
334
- "[13C]": 228,
335
- "[Cl]": 229,
336
- "[PH4+]": 230,
337
- "[SiH4]": 231,
338
- "[te]": 232,
339
- "[CH3-]": 233,
340
- "[S@@+]": 234,
341
- "[Rh+3]": 235,
342
- "[SH+]": 236,
343
- "[Bi+3]": 237,
344
- "[Br+2]": 238,
345
- "[La]": 239,
346
- "[La+3]": 240,
347
- "[Pt-2]": 241,
348
- "[N@@]": 242,
349
- "[PH3+]": 243,
350
- "[N@]": 244,
351
- "[Si+4]": 245,
352
- "[Sr+2]": 246,
353
- "[Al+]": 247,
354
- "[Pb]": 248,
355
- "[SeH]": 249,
356
- "[Si-]": 250,
357
- "[V+5]": 251,
358
- "[Y+3]": 252,
359
- "[Re]": 253,
360
- "[Ru+]": 254,
361
- "[Sm]": 255,
362
- "*": 256,
363
- "[3H]": 257,
364
- "[NH2]": 258,
365
- "[Ag-]": 259,
366
- "[13CH3]": 260,
367
- "[OH+]": 261,
368
- "[Ru+3]": 262,
369
- "[OH]": 263,
370
- "[Gd+3]": 264,
371
- "[13CH2]": 265,
372
- "[In+3]": 266,
373
- "[Si@@]": 267,
374
- "[Si@]": 268,
375
- "[Ti+2]": 269,
376
- "[Sn+]": 270,
377
- "[Cl+2]": 271,
378
- "[AlH-]": 272,
379
- "[Pd-2]": 273,
380
- "[SnH3]": 274,
381
- "[B+3]": 275,
382
- "[Cu-2]": 276,
383
- "[Nd+3]": 277,
384
- "[Pb+3]": 278,
385
- "[13cH]": 279,
386
- "[Fe-4]": 280,
387
- "[Ga]": 281,
388
- "[Sn+4]": 282,
389
- "[Hg+]": 283,
390
- "[11CH3]": 284,
391
- "[Hf]": 285,
392
- "[Pr]": 286,
393
- "[Y]": 287,
394
- "[S+2]": 288,
395
- "[Cd]": 289,
396
- "[Cr+6]": 290,
397
- "[Zr+3]": 291,
398
- "[Rh+]": 292,
399
- "[CH3]": 293,
400
- "[N-3]": 294,
401
- "[Hf+2]": 295,
402
- "[Th]": 296,
403
- "[Sb+3]": 297,
404
- "%14": 298,
405
- "[Cr+2]": 299,
406
- "[Ru+2]": 300,
407
- "[Hf+4]": 301,
408
- "[14C]": 302,
409
- "[Ta]": 303,
410
- "[Tl+]": 304,
411
- "[B+]": 305,
412
- "[Os+4]": 306,
413
- "[PdH2]": 307,
414
- "[Pd-]": 308,
415
- "[Cd+2]": 309,
416
- "[Co+3]": 310,
417
- "[S+4]": 311,
418
- "[Nb+5]": 312,
419
- "[123I]": 313,
420
- "[c+]": 314,
421
- "[Rb+]": 315,
422
- "[V+2]": 316,
423
- "[CH3+]": 317,
424
- "[Ag+2]": 318,
425
- "[cH+]": 319,
426
- "[Mn+3]": 320,
427
- "[Se-]": 321,
428
- "[As-]": 322,
429
- "[Eu+3]": 323,
430
- "[SH2]": 324,
431
- "[Sm+3]": 325,
432
- "[IH+]": 326,
433
- "%15": 327,
434
- "[OH3+]": 328,
435
- "[PH3]": 329,
436
- "[IH2+]": 330,
437
- "[SH2+]": 331,
438
- "[Ir+3]": 332,
439
- "[AlH3]": 333,
440
- "[Sc]": 334,
441
- "[Yb]": 335,
442
- "[15NH2]": 336,
443
- "[Lu]": 337,
444
- "[sH+]": 338,
445
- "[Gd]": 339,
446
- "[18F-]": 340,
447
- "[SH3+]": 341,
448
- "[SnH4]": 342,
449
- "[TeH]": 343,
450
- "[Si@@H]": 344,
451
- "[Ga+3]": 345,
452
- "[CaH2]": 346,
453
- "[Tl]": 347,
454
- "[Ta+5]": 348,
455
- "[GeH]": 349,
456
- "[Br+]": 350,
457
- "[Sr]": 351,
458
- "[Tl+3]": 352,
459
- "[Sm+2]": 353,
460
- "[PH5]": 354,
461
- "%16": 355,
462
- "[N@@+]": 356,
463
- "[Au+3]": 357,
464
- "[C-4]": 358,
465
- "[Nd]": 359,
466
- "[Ti+]": 360,
467
- "[IH]": 361,
468
- "[N@+]": 362,
469
- "[125I]": 363,
470
- "[Eu]": 364,
471
- "[Sn+3]": 365,
472
- "[Nb]": 366,
473
- "[Er+3]": 367,
474
- "[123I-]": 368,
475
- "[14c]": 369,
476
- "%17": 370,
477
- "[SnH2]": 371,
478
- "[YH]": 372,
479
- "[Sb+5]": 373,
480
- "[Pr+3]": 374,
481
- "[Ir+]": 375,
482
- "[N+3]": 376,
483
- "[AlH2]": 377,
484
- "[19F]": 378,
485
- "%18": 379,
486
- "[Tb]": 380,
487
- "[14CH]": 381,
488
- "[Mo+4]": 382,
489
- "[Si+]": 383,
490
- "[BH]": 384,
491
- "[Be]": 385,
492
- "[Rb]": 386,
493
- "[pH]": 387,
494
- "%19": 388,
495
- "%20": 389,
496
- "[Xe]": 390,
497
- "[Ir-]": 391,
498
- "[Be+2]": 392,
499
- "[C+4]": 393,
500
- "[RuH2]": 394,
501
- "[15NH]": 395,
502
- "[U+2]": 396,
503
- "[Au-]": 397,
504
- "%21": 398,
505
- "%22": 399,
506
- "[Au+]": 400,
507
- "[15n]": 401,
508
- "[Al+2]": 402,
509
- "[Tb+3]": 403,
510
- "[15N]": 404,
511
- "[V+3]": 405,
512
- "[W+6]": 406,
513
- "[14CH3]": 407,
514
- "[Cr+4]": 408,
515
- "[ClH+]": 409,
516
- "b": 410,
517
- "[Ti+6]": 411,
518
- "[Nd+]": 412,
519
- "[Zr+]": 413,
520
- "[PH2+]": 414,
521
- "[Fm]": 415,
522
- "[N@H+]": 416,
523
- "[RuH]": 417,
524
- "[Dy+3]": 418,
525
- "%23": 419,
526
- "[Hf+3]": 420,
527
- "[W+4]": 421,
528
- "[11C]": 422,
529
- "[13CH]": 423,
530
- "[Er]": 424,
531
- "[124I]": 425,
532
- "[LaH]": 426,
533
- "[F]": 427,
534
- "[siH]": 428,
535
- "[Ga+]": 429,
536
- "[Cm]": 430,
537
- "[GeH3]": 431,
538
- "[IH-]": 432,
539
- "[U+6]": 433,
540
- "[SeH+]": 434,
541
- "[32P]": 435,
542
- "[SeH-]": 436,
543
- "[Pt-]": 437,
544
- "[Ir+2]": 438,
545
- "[se+]": 439,
546
- "[U]": 440,
547
- "[F+]": 441,
548
- "[BH2]": 442,
549
- "[As+]": 443,
550
- "[Cf]": 444,
551
- "[ClH2+]": 445,
552
- "[Ni+]": 446,
553
- "[TeH3]": 447,
554
- "[SbH2]": 448,
555
- "[Ag+3]": 449,
556
- "%24": 450,
557
- "[18O]": 451,
558
- "[PH4]": 452,
559
- "[Os+2]": 453,
560
- "[Na-]": 454,
561
- "[Sb+2]": 455,
562
- "[V+4]": 456,
563
- "[Ho+3]": 457,
564
- "[68Ga]": 458,
565
- "[PH-]": 459,
566
- "[Bi+2]": 460,
567
- "[Ce+2]": 461,
568
- "[Pd+3]": 462,
569
- "[99Tc]": 463,
570
- "[13C@@H]": 464,
571
- "[Fe+6]": 465,
572
- "[c]": 466,
573
- "[GeH2]": 467,
574
- "[10B]": 468,
575
- "[Cu+3]": 469,
576
- "[Mo+2]": 470,
577
- "[Cr+]": 471,
578
- "[Pd+4]": 472,
579
- "[Dy]": 473,
580
- "[AsH]": 474,
581
- "[Ba+]": 475,
582
- "[SeH2]": 476,
583
- "[In+]": 477,
584
- "[TeH2]": 478,
585
- "[BrH+]": 479,
586
- "[14cH]": 480,
587
- "[W+]": 481,
588
- "[13C@H]": 482,
589
- "[AsH2]": 483,
590
- "[In+2]": 484,
591
- "[N+2]": 485,
592
- "[N@@H+]": 486,
593
- "[SbH]": 487,
594
- "[60Co]": 488,
595
- "[AsH4+]": 489,
596
- "[AsH3]": 490,
597
- "[18OH]": 491,
598
- "[Ru-2]": 492,
599
- "[Na-2]": 493,
600
- "[CuH2]": 494,
601
- "[31P]": 495,
602
- "[Ti+5]": 496,
603
- "[35S]": 497,
604
- "[P@@H]": 498,
605
- "[ArH]": 499,
606
- "[Co+]": 500,
607
- "[Zr-2]": 501,
608
- "[BH2-]": 502,
609
- "[131I]": 503,
610
- "[SH5]": 504,
611
- "[VH]": 505,
612
- "[B+2]": 506,
613
- "[Yb+2]": 507,
614
- "[14C@H]": 508,
615
- "[211At]": 509,
616
- "[NH3+2]": 510,
617
- "[IrH]": 511,
618
- "[IrH2]": 512,
619
- "[Rh-]": 513,
620
- "[Cr-]": 514,
621
- "[Sb+]": 515,
622
- "[Ni+3]": 516,
623
- "[TaH3]": 517,
624
- "[Tl+2]": 518,
625
- "[64Cu]": 519,
626
- "[Tc]": 520,
627
- "[Cd+]": 521,
628
- "[1H]": 522,
629
- "[15nH]": 523,
630
- "[AlH2+]": 524,
631
- "[FH+2]": 525,
632
- "[BiH3]": 526,
633
- "[Ru-]": 527,
634
- "[Mo+6]": 528,
635
- "[AsH+]": 529,
636
- "[BaH2]": 530,
637
- "[BaH]": 531,
638
- "[Fe+4]": 532,
639
- "[229Th]": 533,
640
- "[Th+4]": 534,
641
- "[As+3]": 535,
642
- "[NH+3]": 536,
643
- "[P@H]": 537,
644
- "[Li-]": 538,
645
- "[7NaH]": 539,
646
- "[Bi+]": 540,
647
- "[PtH+2]": 541,
648
- "[p-]": 542,
649
- "[Re+5]": 543,
650
- "[NiH]": 544,
651
- "[Ni-]": 545,
652
- "[Xe+]": 546,
653
- "[Ca+]": 547,
654
- "[11c]": 548,
655
- "[Rh+4]": 549,
656
- "[AcH]": 550,
657
- "[HeH]": 551,
658
- "[Sc+2]": 552,
659
- "[Mn+]": 553,
660
- "[UH]": 554,
661
- "[14CH2]": 555,
662
- "[SiH4+]": 556,
663
- "[18OH2]": 557,
664
- "[Ac-]": 558,
665
- "[Re+4]": 559,
666
- "[118Sn]": 560,
667
- "[153Sm]": 561,
668
- "[P+2]": 562,
669
- "[9CH]": 563,
670
- "[9CH3]": 564,
671
- "[Y-]": 565,
672
- "[NiH2]": 566,
673
- "[Si+2]": 567,
674
- "[Mn+6]": 568,
675
- "[ZrH2]": 569,
676
- "[C-2]": 570,
677
- "[Bi+5]": 571,
678
- "[24NaH]": 572,
679
- "[Fr]": 573,
680
- "[15CH]": 574,
681
- "[Se+]": 575,
682
- "[At]": 576,
683
- "[P-3]": 577,
684
- "[124I-]": 578,
685
- "[CuH2-]": 579,
686
- "[Nb+4]": 580,
687
- "[Nb+3]": 581,
688
- "[MgH]": 582,
689
- "[Ir+4]": 583,
690
- "[67Ga+3]": 584,
691
- "[67Ga]": 585,
692
- "[13N]": 586,
693
- "[15OH2]": 587,
694
- "[2NH]": 588,
695
- "[Ho]": 589,
696
- "[Cn]": 590
697
- },
698
- "merges": []
699
- }
700
- }