omshrivastava commited on
Commit
67627d8
·
verified ·
1 Parent(s): 15fb647

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +705 -0
  2. tokenizer_config.json +23 -0
tokenizer.json ADDED
@@ -0,0 +1,705 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": {
4
+ "direction": "Right",
5
+ "max_length": 256,
6
+ "strategy": "LongestFirst",
7
+ "stride": 0
8
+ },
9
+ "padding": null,
10
+ "added_tokens": [
11
+ {
12
+ "id": 0,
13
+ "content": "[PAD]",
14
+ "single_word": false,
15
+ "lstrip": false,
16
+ "rstrip": false,
17
+ "normalized": false,
18
+ "special": true
19
+ },
20
+ {
21
+ "id": 11,
22
+ "content": "[UNK]",
23
+ "single_word": false,
24
+ "lstrip": false,
25
+ "rstrip": false,
26
+ "normalized": false,
27
+ "special": true
28
+ },
29
+ {
30
+ "id": 12,
31
+ "content": "[CLS]",
32
+ "single_word": false,
33
+ "lstrip": false,
34
+ "rstrip": false,
35
+ "normalized": false,
36
+ "special": true
37
+ },
38
+ {
39
+ "id": 13,
40
+ "content": "[SEP]",
41
+ "single_word": false,
42
+ "lstrip": false,
43
+ "rstrip": false,
44
+ "normalized": false,
45
+ "special": true
46
+ },
47
+ {
48
+ "id": 14,
49
+ "content": "[MASK]",
50
+ "single_word": false,
51
+ "lstrip": true,
52
+ "rstrip": false,
53
+ "normalized": false,
54
+ "special": true
55
+ },
56
+ {
57
+ "id": 591,
58
+ "content": "<s>",
59
+ "single_word": false,
60
+ "lstrip": false,
61
+ "rstrip": false,
62
+ "normalized": true,
63
+ "special": true
64
+ },
65
+ {
66
+ "id": 592,
67
+ "content": "</s>",
68
+ "single_word": false,
69
+ "lstrip": false,
70
+ "rstrip": false,
71
+ "normalized": true,
72
+ "special": true
73
+ }
74
+ ],
75
+ "normalizer": null,
76
+ "pre_tokenizer": {
77
+ "type": "ByteLevel",
78
+ "add_prefix_space": false,
79
+ "trim_offsets": true,
80
+ "use_regex": true
81
+ },
82
+ "post_processor": {
83
+ "type": "RobertaProcessing",
84
+ "sep": [
85
+ "[SEP]",
86
+ 13
87
+ ],
88
+ "cls": [
89
+ "[CLS]",
90
+ 12
91
+ ],
92
+ "trim_offsets": true,
93
+ "add_prefix_space": false
94
+ },
95
+ "decoder": {
96
+ "type": "ByteLevel",
97
+ "add_prefix_space": true,
98
+ "trim_offsets": true,
99
+ "use_regex": true
100
+ },
101
+ "model": {
102
+ "type": "BPE",
103
+ "dropout": null,
104
+ "unk_token": null,
105
+ "continuing_subword_prefix": "",
106
+ "end_of_word_suffix": "",
107
+ "fuse_unk": false,
108
+ "byte_fallback": false,
109
+ "ignore_merges": false,
110
+ "vocab": {
111
+ "[PAD]": 0,
112
+ "[unused1]": 1,
113
+ "[unused2]": 2,
114
+ "[unused3]": 3,
115
+ "[unused4]": 4,
116
+ "[unused5]": 5,
117
+ "[unused6]": 6,
118
+ "[unused7]": 7,
119
+ "[unused8]": 8,
120
+ "[unused9]": 9,
121
+ "[unused10]": 10,
122
+ "[UNK]": 11,
123
+ "[CLS]": 12,
124
+ "[SEP]": 13,
125
+ "[MASK]": 14,
126
+ "c": 15,
127
+ "C": 16,
128
+ "(": 17,
129
+ ")": 18,
130
+ "O": 19,
131
+ "1": 20,
132
+ "2": 21,
133
+ "=": 22,
134
+ "N": 23,
135
+ ".": 24,
136
+ "n": 25,
137
+ "3": 26,
138
+ "F": 27,
139
+ "Cl": 28,
140
+ ">>": 29,
141
+ "~": 30,
142
+ "-": 31,
143
+ "4": 32,
144
+ "[C@H]": 33,
145
+ "S": 34,
146
+ "[C@@H]": 35,
147
+ "[O-]": 36,
148
+ "Br": 37,
149
+ "#": 38,
150
+ "/": 39,
151
+ "[nH]": 40,
152
+ "[N+]": 41,
153
+ "s": 42,
154
+ "5": 43,
155
+ "o": 44,
156
+ "P": 45,
157
+ "[Na+]": 46,
158
+ "[Si]": 47,
159
+ "I": 48,
160
+ "[Na]": 49,
161
+ "[Pd]": 50,
162
+ "[K+]": 51,
163
+ "[K]": 52,
164
+ "[P]": 53,
165
+ "B": 54,
166
+ "[C@]": 55,
167
+ "[C@@]": 56,
168
+ "[Cl-]": 57,
169
+ "6": 58,
170
+ "[OH-]": 59,
171
+ "\\": 60,
172
+ "[N-]": 61,
173
+ "[Li]": 62,
174
+ "[H]": 63,
175
+ "[2H]": 64,
176
+ "[NH4+]": 65,
177
+ "[c-]": 66,
178
+ "[P-]": 67,
179
+ "[Cs+]": 68,
180
+ "[Li+]": 69,
181
+ "[Cs]": 70,
182
+ "[NaH]": 71,
183
+ "[H-]": 72,
184
+ "[O+]": 73,
185
+ "[BH4-]": 74,
186
+ "[Cu]": 75,
187
+ "7": 76,
188
+ "[Mg]": 77,
189
+ "[Fe+2]": 78,
190
+ "[n+]": 79,
191
+ "[Sn]": 80,
192
+ "[BH-]": 81,
193
+ "[Pd+2]": 82,
194
+ "[CH]": 83,
195
+ "[I-]": 84,
196
+ "[Br-]": 85,
197
+ "[C-]": 86,
198
+ "[Zn]": 87,
199
+ "[B-]": 88,
200
+ "[F-]": 89,
201
+ "[Al]": 90,
202
+ "[P+]": 91,
203
+ "[BH3-]": 92,
204
+ "[Fe]": 93,
205
+ "[C]": 94,
206
+ "[AlH4]": 95,
207
+ "[Ni]": 96,
208
+ "[SiH]": 97,
209
+ "8": 98,
210
+ "[Cu+2]": 99,
211
+ "[Mn]": 100,
212
+ "[AlH]": 101,
213
+ "[nH+]": 102,
214
+ "[AlH4-]": 103,
215
+ "[O-2]": 104,
216
+ "[Cr]": 105,
217
+ "[Mg+2]": 106,
218
+ "[NH3+]": 107,
219
+ "[S@]": 108,
220
+ "[Pt]": 109,
221
+ "[Al+3]": 110,
222
+ "[S@@]": 111,
223
+ "[S-]": 112,
224
+ "[Ti]": 113,
225
+ "[Zn+2]": 114,
226
+ "[PH]": 115,
227
+ "[NH2+]": 116,
228
+ "[Ru]": 117,
229
+ "[Ag+]": 118,
230
+ "[S+]": 119,
231
+ "[I+3]": 120,
232
+ "[NH+]": 121,
233
+ "[Ca+2]": 122,
234
+ "[Ag]": 123,
235
+ "9": 124,
236
+ "[Os]": 125,
237
+ "[Se]": 126,
238
+ "[SiH2]": 127,
239
+ "[Ca]": 128,
240
+ "[Ti+4]": 129,
241
+ "[Ac]": 130,
242
+ "[Cu+]": 131,
243
+ "[S]": 132,
244
+ "[Rh]": 133,
245
+ "[Cl+3]": 134,
246
+ "[cH-]": 135,
247
+ "[Zn+]": 136,
248
+ "[O]": 137,
249
+ "[Cl+]": 138,
250
+ "[SH]": 139,
251
+ "[H+]": 140,
252
+ "[Pd+]": 141,
253
+ "[se]": 142,
254
+ "[PH+]": 143,
255
+ "[I]": 144,
256
+ "[Pt+2]": 145,
257
+ "[C+]": 146,
258
+ "[Mg+]": 147,
259
+ "[Hg]": 148,
260
+ "[W]": 149,
261
+ "[SnH]": 150,
262
+ "[SiH3]": 151,
263
+ "[Fe+3]": 152,
264
+ "[NH]": 153,
265
+ "[Mo]": 154,
266
+ "[CH2+]": 155,
267
+ "%10": 156,
268
+ "[CH2-]": 157,
269
+ "[CH2]": 158,
270
+ "[n-]": 159,
271
+ "[Ce+4]": 160,
272
+ "[NH-]": 161,
273
+ "[Co]": 162,
274
+ "[I+]": 163,
275
+ "[PH2]": 164,
276
+ "[Pt+4]": 165,
277
+ "[Ce]": 166,
278
+ "[B]": 167,
279
+ "[Sn+2]": 168,
280
+ "[Ba+2]": 169,
281
+ "%11": 170,
282
+ "[Fe-3]": 171,
283
+ "[18F]": 172,
284
+ "[SH-]": 173,
285
+ "[Pb+2]": 174,
286
+ "[Os-2]": 175,
287
+ "[Zr+4]": 176,
288
+ "[N]": 177,
289
+ "[Ir]": 178,
290
+ "[Bi]": 179,
291
+ "[Ni+2]": 180,
292
+ "[P@]": 181,
293
+ "[Co+2]": 182,
294
+ "[s+]": 183,
295
+ "[As]": 184,
296
+ "[P+3]": 185,
297
+ "[Hg+2]": 186,
298
+ "[Yb+3]": 187,
299
+ "[CH-]": 188,
300
+ "[Zr+2]": 189,
301
+ "[Mn+2]": 190,
302
+ "[CH+]": 191,
303
+ "[In]": 192,
304
+ "[KH]": 193,
305
+ "[Ce+3]": 194,
306
+ "[Zr]": 195,
307
+ "[AlH2-]": 196,
308
+ "[OH2+]": 197,
309
+ "[Ti+3]": 198,
310
+ "[Rh+2]": 199,
311
+ "[Sb]": 200,
312
+ "[S-2]": 201,
313
+ "%12": 202,
314
+ "[P@@]": 203,
315
+ "[Si@H]": 204,
316
+ "[Mn+4]": 205,
317
+ "p": 206,
318
+ "[Ba]": 207,
319
+ "[NH2-]": 208,
320
+ "[Ge]": 209,
321
+ "[Pb+4]": 210,
322
+ "[Cr+3]": 211,
323
+ "[Au]": 212,
324
+ "[LiH]": 213,
325
+ "[Sc+3]": 214,
326
+ "[o+]": 215,
327
+ "[Rh-3]": 216,
328
+ "%13": 217,
329
+ "[Br]": 218,
330
+ "[Sb-]": 219,
331
+ "[S@+]": 220,
332
+ "[I+2]": 221,
333
+ "[Ar]": 222,
334
+ "[V]": 223,
335
+ "[Cu-]": 224,
336
+ "[Al-]": 225,
337
+ "[Te]": 226,
338
+ "[13c]": 227,
339
+ "[13C]": 228,
340
+ "[Cl]": 229,
341
+ "[PH4+]": 230,
342
+ "[SiH4]": 231,
343
+ "[te]": 232,
344
+ "[CH3-]": 233,
345
+ "[S@@+]": 234,
346
+ "[Rh+3]": 235,
347
+ "[SH+]": 236,
348
+ "[Bi+3]": 237,
349
+ "[Br+2]": 238,
350
+ "[La]": 239,
351
+ "[La+3]": 240,
352
+ "[Pt-2]": 241,
353
+ "[N@@]": 242,
354
+ "[PH3+]": 243,
355
+ "[N@]": 244,
356
+ "[Si+4]": 245,
357
+ "[Sr+2]": 246,
358
+ "[Al+]": 247,
359
+ "[Pb]": 248,
360
+ "[SeH]": 249,
361
+ "[Si-]": 250,
362
+ "[V+5]": 251,
363
+ "[Y+3]": 252,
364
+ "[Re]": 253,
365
+ "[Ru+]": 254,
366
+ "[Sm]": 255,
367
+ "*": 256,
368
+ "[3H]": 257,
369
+ "[NH2]": 258,
370
+ "[Ag-]": 259,
371
+ "[13CH3]": 260,
372
+ "[OH+]": 261,
373
+ "[Ru+3]": 262,
374
+ "[OH]": 263,
375
+ "[Gd+3]": 264,
376
+ "[13CH2]": 265,
377
+ "[In+3]": 266,
378
+ "[Si@@]": 267,
379
+ "[Si@]": 268,
380
+ "[Ti+2]": 269,
381
+ "[Sn+]": 270,
382
+ "[Cl+2]": 271,
383
+ "[AlH-]": 272,
384
+ "[Pd-2]": 273,
385
+ "[SnH3]": 274,
386
+ "[B+3]": 275,
387
+ "[Cu-2]": 276,
388
+ "[Nd+3]": 277,
389
+ "[Pb+3]": 278,
390
+ "[13cH]": 279,
391
+ "[Fe-4]": 280,
392
+ "[Ga]": 281,
393
+ "[Sn+4]": 282,
394
+ "[Hg+]": 283,
395
+ "[11CH3]": 284,
396
+ "[Hf]": 285,
397
+ "[Pr]": 286,
398
+ "[Y]": 287,
399
+ "[S+2]": 288,
400
+ "[Cd]": 289,
401
+ "[Cr+6]": 290,
402
+ "[Zr+3]": 291,
403
+ "[Rh+]": 292,
404
+ "[CH3]": 293,
405
+ "[N-3]": 294,
406
+ "[Hf+2]": 295,
407
+ "[Th]": 296,
408
+ "[Sb+3]": 297,
409
+ "%14": 298,
410
+ "[Cr+2]": 299,
411
+ "[Ru+2]": 300,
412
+ "[Hf+4]": 301,
413
+ "[14C]": 302,
414
+ "[Ta]": 303,
415
+ "[Tl+]": 304,
416
+ "[B+]": 305,
417
+ "[Os+4]": 306,
418
+ "[PdH2]": 307,
419
+ "[Pd-]": 308,
420
+ "[Cd+2]": 309,
421
+ "[Co+3]": 310,
422
+ "[S+4]": 311,
423
+ "[Nb+5]": 312,
424
+ "[123I]": 313,
425
+ "[c+]": 314,
426
+ "[Rb+]": 315,
427
+ "[V+2]": 316,
428
+ "[CH3+]": 317,
429
+ "[Ag+2]": 318,
430
+ "[cH+]": 319,
431
+ "[Mn+3]": 320,
432
+ "[Se-]": 321,
433
+ "[As-]": 322,
434
+ "[Eu+3]": 323,
435
+ "[SH2]": 324,
436
+ "[Sm+3]": 325,
437
+ "[IH+]": 326,
438
+ "%15": 327,
439
+ "[OH3+]": 328,
440
+ "[PH3]": 329,
441
+ "[IH2+]": 330,
442
+ "[SH2+]": 331,
443
+ "[Ir+3]": 332,
444
+ "[AlH3]": 333,
445
+ "[Sc]": 334,
446
+ "[Yb]": 335,
447
+ "[15NH2]": 336,
448
+ "[Lu]": 337,
449
+ "[sH+]": 338,
450
+ "[Gd]": 339,
451
+ "[18F-]": 340,
452
+ "[SH3+]": 341,
453
+ "[SnH4]": 342,
454
+ "[TeH]": 343,
455
+ "[Si@@H]": 344,
456
+ "[Ga+3]": 345,
457
+ "[CaH2]": 346,
458
+ "[Tl]": 347,
459
+ "[Ta+5]": 348,
460
+ "[GeH]": 349,
461
+ "[Br+]": 350,
462
+ "[Sr]": 351,
463
+ "[Tl+3]": 352,
464
+ "[Sm+2]": 353,
465
+ "[PH5]": 354,
466
+ "%16": 355,
467
+ "[N@@+]": 356,
468
+ "[Au+3]": 357,
469
+ "[C-4]": 358,
470
+ "[Nd]": 359,
471
+ "[Ti+]": 360,
472
+ "[IH]": 361,
473
+ "[N@+]": 362,
474
+ "[125I]": 363,
475
+ "[Eu]": 364,
476
+ "[Sn+3]": 365,
477
+ "[Nb]": 366,
478
+ "[Er+3]": 367,
479
+ "[123I-]": 368,
480
+ "[14c]": 369,
481
+ "%17": 370,
482
+ "[SnH2]": 371,
483
+ "[YH]": 372,
484
+ "[Sb+5]": 373,
485
+ "[Pr+3]": 374,
486
+ "[Ir+]": 375,
487
+ "[N+3]": 376,
488
+ "[AlH2]": 377,
489
+ "[19F]": 378,
490
+ "%18": 379,
491
+ "[Tb]": 380,
492
+ "[14CH]": 381,
493
+ "[Mo+4]": 382,
494
+ "[Si+]": 383,
495
+ "[BH]": 384,
496
+ "[Be]": 385,
497
+ "[Rb]": 386,
498
+ "[pH]": 387,
499
+ "%19": 388,
500
+ "%20": 389,
501
+ "[Xe]": 390,
502
+ "[Ir-]": 391,
503
+ "[Be+2]": 392,
504
+ "[C+4]": 393,
505
+ "[RuH2]": 394,
506
+ "[15NH]": 395,
507
+ "[U+2]": 396,
508
+ "[Au-]": 397,
509
+ "%21": 398,
510
+ "%22": 399,
511
+ "[Au+]": 400,
512
+ "[15n]": 401,
513
+ "[Al+2]": 402,
514
+ "[Tb+3]": 403,
515
+ "[15N]": 404,
516
+ "[V+3]": 405,
517
+ "[W+6]": 406,
518
+ "[14CH3]": 407,
519
+ "[Cr+4]": 408,
520
+ "[ClH+]": 409,
521
+ "b": 410,
522
+ "[Ti+6]": 411,
523
+ "[Nd+]": 412,
524
+ "[Zr+]": 413,
525
+ "[PH2+]": 414,
526
+ "[Fm]": 415,
527
+ "[N@H+]": 416,
528
+ "[RuH]": 417,
529
+ "[Dy+3]": 418,
530
+ "%23": 419,
531
+ "[Hf+3]": 420,
532
+ "[W+4]": 421,
533
+ "[11C]": 422,
534
+ "[13CH]": 423,
535
+ "[Er]": 424,
536
+ "[124I]": 425,
537
+ "[LaH]": 426,
538
+ "[F]": 427,
539
+ "[siH]": 428,
540
+ "[Ga+]": 429,
541
+ "[Cm]": 430,
542
+ "[GeH3]": 431,
543
+ "[IH-]": 432,
544
+ "[U+6]": 433,
545
+ "[SeH+]": 434,
546
+ "[32P]": 435,
547
+ "[SeH-]": 436,
548
+ "[Pt-]": 437,
549
+ "[Ir+2]": 438,
550
+ "[se+]": 439,
551
+ "[U]": 440,
552
+ "[F+]": 441,
553
+ "[BH2]": 442,
554
+ "[As+]": 443,
555
+ "[Cf]": 444,
556
+ "[ClH2+]": 445,
557
+ "[Ni+]": 446,
558
+ "[TeH3]": 447,
559
+ "[SbH2]": 448,
560
+ "[Ag+3]": 449,
561
+ "%24": 450,
562
+ "[18O]": 451,
563
+ "[PH4]": 452,
564
+ "[Os+2]": 453,
565
+ "[Na-]": 454,
566
+ "[Sb+2]": 455,
567
+ "[V+4]": 456,
568
+ "[Ho+3]": 457,
569
+ "[68Ga]": 458,
570
+ "[PH-]": 459,
571
+ "[Bi+2]": 460,
572
+ "[Ce+2]": 461,
573
+ "[Pd+3]": 462,
574
+ "[99Tc]": 463,
575
+ "[13C@@H]": 464,
576
+ "[Fe+6]": 465,
577
+ "[c]": 466,
578
+ "[GeH2]": 467,
579
+ "[10B]": 468,
580
+ "[Cu+3]": 469,
581
+ "[Mo+2]": 470,
582
+ "[Cr+]": 471,
583
+ "[Pd+4]": 472,
584
+ "[Dy]": 473,
585
+ "[AsH]": 474,
586
+ "[Ba+]": 475,
587
+ "[SeH2]": 476,
588
+ "[In+]": 477,
589
+ "[TeH2]": 478,
590
+ "[BrH+]": 479,
591
+ "[14cH]": 480,
592
+ "[W+]": 481,
593
+ "[13C@H]": 482,
594
+ "[AsH2]": 483,
595
+ "[In+2]": 484,
596
+ "[N+2]": 485,
597
+ "[N@@H+]": 486,
598
+ "[SbH]": 487,
599
+ "[60Co]": 488,
600
+ "[AsH4+]": 489,
601
+ "[AsH3]": 490,
602
+ "[18OH]": 491,
603
+ "[Ru-2]": 492,
604
+ "[Na-2]": 493,
605
+ "[CuH2]": 494,
606
+ "[31P]": 495,
607
+ "[Ti+5]": 496,
608
+ "[35S]": 497,
609
+ "[P@@H]": 498,
610
+ "[ArH]": 499,
611
+ "[Co+]": 500,
612
+ "[Zr-2]": 501,
613
+ "[BH2-]": 502,
614
+ "[131I]": 503,
615
+ "[SH5]": 504,
616
+ "[VH]": 505,
617
+ "[B+2]": 506,
618
+ "[Yb+2]": 507,
619
+ "[14C@H]": 508,
620
+ "[211At]": 509,
621
+ "[NH3+2]": 510,
622
+ "[IrH]": 511,
623
+ "[IrH2]": 512,
624
+ "[Rh-]": 513,
625
+ "[Cr-]": 514,
626
+ "[Sb+]": 515,
627
+ "[Ni+3]": 516,
628
+ "[TaH3]": 517,
629
+ "[Tl+2]": 518,
630
+ "[64Cu]": 519,
631
+ "[Tc]": 520,
632
+ "[Cd+]": 521,
633
+ "[1H]": 522,
634
+ "[15nH]": 523,
635
+ "[AlH2+]": 524,
636
+ "[FH+2]": 525,
637
+ "[BiH3]": 526,
638
+ "[Ru-]": 527,
639
+ "[Mo+6]": 528,
640
+ "[AsH+]": 529,
641
+ "[BaH2]": 530,
642
+ "[BaH]": 531,
643
+ "[Fe+4]": 532,
644
+ "[229Th]": 533,
645
+ "[Th+4]": 534,
646
+ "[As+3]": 535,
647
+ "[NH+3]": 536,
648
+ "[P@H]": 537,
649
+ "[Li-]": 538,
650
+ "[7NaH]": 539,
651
+ "[Bi+]": 540,
652
+ "[PtH+2]": 541,
653
+ "[p-]": 542,
654
+ "[Re+5]": 543,
655
+ "[NiH]": 544,
656
+ "[Ni-]": 545,
657
+ "[Xe+]": 546,
658
+ "[Ca+]": 547,
659
+ "[11c]": 548,
660
+ "[Rh+4]": 549,
661
+ "[AcH]": 550,
662
+ "[HeH]": 551,
663
+ "[Sc+2]": 552,
664
+ "[Mn+]": 553,
665
+ "[UH]": 554,
666
+ "[14CH2]": 555,
667
+ "[SiH4+]": 556,
668
+ "[18OH2]": 557,
669
+ "[Ac-]": 558,
670
+ "[Re+4]": 559,
671
+ "[118Sn]": 560,
672
+ "[153Sm]": 561,
673
+ "[P+2]": 562,
674
+ "[9CH]": 563,
675
+ "[9CH3]": 564,
676
+ "[Y-]": 565,
677
+ "[NiH2]": 566,
678
+ "[Si+2]": 567,
679
+ "[Mn+6]": 568,
680
+ "[ZrH2]": 569,
681
+ "[C-2]": 570,
682
+ "[Bi+5]": 571,
683
+ "[24NaH]": 572,
684
+ "[Fr]": 573,
685
+ "[15CH]": 574,
686
+ "[Se+]": 575,
687
+ "[At]": 576,
688
+ "[P-3]": 577,
689
+ "[124I-]": 578,
690
+ "[CuH2-]": 579,
691
+ "[Nb+4]": 580,
692
+ "[Nb+3]": 581,
693
+ "[MgH]": 582,
694
+ "[Ir+4]": 583,
695
+ "[67Ga+3]": 584,
696
+ "[67Ga]": 585,
697
+ "[13N]": 586,
698
+ "[15OH2]": 587,
699
+ "[2NH]": 588,
700
+ "[Ho]": 589,
701
+ "[Cn]": 590
702
+ },
703
+ "merges": []
704
+ }
705
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<s>",
5
+ "cls_token": "[CLS]",
6
+ "eos_token": "</s>",
7
+ "errors": "replace",
8
+ "full_tokenizer_file": null,
9
+ "is_local": true,
10
+ "local_files_only": false,
11
+ "mask_token": "[MASK]",
12
+ "max_len": 512,
13
+ "max_length": 256,
14
+ "model_max_length": 512,
15
+ "pad_token": "[PAD]",
16
+ "sep_token": "[SEP]",
17
+ "stride": 0,
18
+ "tokenizer_class": "RobertaTokenizer",
19
+ "trim_offsets": true,
20
+ "truncation_side": "right",
21
+ "truncation_strategy": "longest_first",
22
+ "unk_token": "[UNK]"
23
+ }