nuevamol-bpe-tokenizer / tokenizer.json
ddidacus's picture
Upload folder using huggingface_hub
ff9cd5f verified
Raw
History Blame Contribute Delete
23.5 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 387,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 388,
"content": "[EOS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 389,
"content": "[BOS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 390,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "Split",
"pattern": {
"Regex": "(\\[[^\\]]+]|Br?|Cl?|N|O|S|P|F|I|b|c|n|o|s|p|\\(|\\)|\\.|=|#|\\+|\\/|:|@|\\?|>|\\*|\\$|%[0-9]{2}|[0-9])"
},
"behavior": "Isolated",
"invert": false
},
"post_processor": null,
"decoder": {
"type": "Fuse"
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"#": 0,
"%": 1,
"(": 2,
")": 3,
"+": 4,
"-": 5,
".": 6,
"/": 7,
"0": 8,
"1": 9,
"2": 10,
"3": 11,
"4": 12,
"5": 13,
"6": 14,
"7": 15,
"8": 16,
"9": 17,
"=": 18,
"@": 19,
"B": 20,
"C": 21,
"F": 22,
"H": 23,
"I": 24,
"N": 25,
"O": 26,
"P": 27,
"S": 28,
"T": 29,
"Y": 30,
"Z": 31,
"[": 32,
"\\": 33,
"]": 34,
"b": 35,
"c": 36,
"e": 37,
"i": 38,
"l": 39,
"n": 40,
"o": 41,
"p": 42,
"r": 43,
"s": 44,
"H]": 45,
"[C": 46,
"[C@": 47,
"Cl": 48,
"[C@@": 49,
"[C@H]": 50,
"[C@@H]": 51,
"[n": 52,
"[nH]": 53,
"[C@]": 54,
"[C@@]": 55,
"%1": 56,
"[CH]": 57,
"%2": 58,
"[C]": 59,
"%10": 60,
"%3": 61,
"%11": 62,
"Br": 63,
"%4": 64,
"%12": 65,
"%13": 66,
"%14": 67,
"[N": 68,
"%5": 69,
"%15": 70,
"%16": 71,
"%17": 72,
"2]": 73,
"H2]": 74,
"[CH2]": 75,
"%18": 76,
"%19": 77,
"%20": 78,
"[c": 79,
"%21": 80,
"[c]": 81,
"%22": 82,
"%23": 83,
"%6": 84,
"%24": 85,
"[1": 86,
"%25": 87,
"%26": 88,
"+]": 89,
"%27": 90,
"[p": 91,
"[pH]": 92,
"[N]": 93,
"%28": 94,
"%29": 95,
"[18": 96,
"%30": 97,
"F]": 98,
"%31": 99,
"[18F]": 100,
"%32": 101,
"[N+]": 102,
"%33": 103,
"%34": 104,
"%35": 105,
"%36": 106,
"%37": 107,
"%38": 108,
"%39": 109,
"%7": 110,
"%40": 111,
"%41": 112,
"@]": 113,
"%42": 114,
"%43": 115,
"O]": 116,
"%44": 117,
"%45": 118,
"[O]": 119,
"%46": 120,
"%47": 121,
"%48": 122,
"[S": 123,
"[NH]": 124,
"%49": 125,
"%50": 126,
"%8": 127,
"%51": 128,
"@@]": 129,
"%52": 130,
"%53": 131,
"%54": 132,
"[n+]": 133,
"%55": 134,
"%56": 135,
"%57": 136,
"%58": 137,
"%59": 138,
"%60": 139,
"%9": 140,
"[S@@]": 141,
"%61": 142,
"%62": 143,
"%63": 144,
"[N@@]": 145,
"%64": 146,
"%65": 147,
"%66": 148,
"[N@]": 149,
"[15": 150,
"[S@]": 151,
"[15n": 152,
"%67": 153,
"[H": 154,
"%68": 155,
"[HH]": 156,
"H+]": 157,
"-]": 158,
"%69": 159,
"%70": 160,
"[15n]": 161,
"%71": 162,
"%72": 163,
"%73": 164,
"%74": 165,
"%75": 166,
"%76": 167,
"[S]": 168,
"%77": 169,
"%78": 170,
"[15nH]": 171,
"[nH+]": 172,
"%79": 173,
"[18O]": 174,
"OH]": 175,
"%80": 176,
"%81": 177,
"[NH+]": 178,
"%82": 179,
"%83": 180,
"[12": 181,
"[N-]": 182,
"%84": 183,
"[18OH]": 184,
"%85": 185,
"Y]": 186,
"[s": 187,
"[Y]": 188,
"[c-]": 189,
"%86": 190,
"[si": 191,
"%89": 192,
"%88": 193,
"%90": 194,
"%87": 195,
"[12C": 196,
"%91": 197,
"-2]": 198,
"[1H]": 199,
"%92": 200,
"[17": 201,
"[19": 202,
"[S-2]": 203,
"[si+]": 204,
"[19F]": 205,
"%93": 206,
"3]": 207,
"[3": 208,
"H3]": 209,
"%94": 210,
"[n-]": 211,
"%95": 212,
"%96": 213,
"[12c": 214,
"%97": 215,
"%98": 216,
"%99": 217,
"[Si": 218,
"[17O]": 219,
"[Si]": 220,
"[12cH]": 221,
"Cl]": 222,
"NH2]": 223,
"[si]": 224,
"[14": 225,
"[16": 226,
"[12CH2]": 227,
"[17OH]": 228,
"[35": 229,
"[12CH3]": 230,
"[35Cl]": 231,
"[SH]": 232,
"0C": 233,
"N]": 234,
"O-]": 235,
"[O-]": 236,
"[12C]": 237,
"[CH3]": 238,
"[13": 239,
"+2]": 240,
"4]": 241,
"H4]": 242,
"[2": 243,
"[B": 244,
"CH2]": 245,
"FH]": 246,
"SH2]": 247,
"[9": 248,
"[34": 249,
"[12c]": 250,
"[14N]": 251,
"[16NH2]": 252,
"[13NH2]": 253,
"[B]": 254,
"S]": 255,
"[8": 256,
"c]": 257,
"[10C": 258,
"[SH2]": 259,
"[H+]": 260,
"[12CH]": 261,
"@H]": 262,
"C]": 263,
"CH3]": 264,
"CH4]": 265,
"NH]": 266,
"cH]": 267,
"[18FH]": 268,
"[20C": 269,
"CH]": 270,
"Zn": 271,
"[Zn": 272,
"n]": 273,
"nH]": 274,
"o]": 275,
"s]": 276,
"[C-]": 277,
"[N-2]": 278,
"[11": 279,
"[1C]": 280,
"[18o]": 281,
"[S@": 282,
"[siH+]": 283,
"[37": 284,
"[SiH3]": 285,
"[14NH]": 286,
"[34S]": 287,
"[Zn+2]": 288,
"2CH4]": 289,
"@@H]": 290,
"HH]": 291,
"TB": 292,
"[C+2]": 293,
"[NH": 294,
"[N+2]": 295,
"[1HH]": 296,
"[18c]": 297,
"[18n]": 298,
"[15CH2]": 299,
"[siH]": 300,
"[12CH4]": 301,
"[12C@H]": 302,
"[12C@@H]": 303,
"[32": 304,
"[33": 305,
"[36": 306,
"[38": 307,
"[14NH2]": 308,
"[16OH]": 309,
"[16N]": 310,
"[16nH]": 311,
"[35s]": 312,
"[22CH4]": 313,
"[9CH2]": 314,
"[9c]": 315,
"[8CH2]": 316,
"[8CH]": 317,
"[10CH2]": 318,
"[20C]": 319,
"[20CH3]": 320,
"[S@TB": 321,
"[37Cl]": 322,
"0cH]": 323,
"1]": 324,
"1CH3]": 325,
"2+]": 326,
"3CH3]": 327,
"4CH3]": 328,
"5]": 329,
"Fe": 330,
"PH]": 331,
"P1]": 332,
"S-2]": 333,
"SP1]": 334,
"[H]": 335,
"[F]": 336,
"[Fe": 337,
"[PH]": 338,
"l-]": 339,
"[Cl-]": 340,
"[1H+]": 341,
"[1CH3]": 342,
"[15OH]": 343,
"[15c]": 344,
"[15CH4]": 345,
"[15cH]": 346,
"[15CH]": 347,
"[si-]": 348,
"[17F]": 349,
"[17FH]": 350,
"[19FH]": 351,
"[3C]": 352,
"[14n]": 353,
"[14nH]": 354,
"[16NH]": 355,
"[13O]": 356,
"[13N]": 357,
"[20cH]": 358,
"[B-]": 359,
"[9CH4]": 360,
"[9cH]": 361,
"[34SH2]": 362,
"[34s]": 363,
"[34S-2]": 364,
"[8cH]": 365,
"[10C]": 366,
"[10CH3]": 367,
"[10CH4]": 368,
"[111CH3]": 369,
"[113CH3]": 370,
"[114CH3]": 371,
"[S@SP1]": 372,
"[37SH2]": 373,
"[NH-]": 374,
"[NH2+]": 375,
"[32SH2]": 376,
"[32S]": 377,
"[33SH2]": 378,
"[33S]": 379,
"[36Cl]": 380,
"[36SH2]": 381,
"[38Cl]": 382,
"[38SH2]": 383,
"[S@TB3]": 384,
"[S@TB5]": 385,
"[Fe]": 386
},
"merges": [
[
"H",
"]"
],
[
"[",
"C"
],
[
"[C",
"@"
],
[
"C",
"l"
],
[
"[C@",
"@"
],
[
"[C@",
"H]"
],
[
"[C@@",
"H]"
],
[
"[",
"n"
],
[
"[n",
"H]"
],
[
"[C@",
"]"
],
[
"[C@@",
"]"
],
[
"%",
"1"
],
[
"[C",
"H]"
],
[
"%",
"2"
],
[
"[C",
"]"
],
[
"%1",
"0"
],
[
"%",
"3"
],
[
"%1",
"1"
],
[
"B",
"r"
],
[
"%",
"4"
],
[
"%1",
"2"
],
[
"%1",
"3"
],
[
"%1",
"4"
],
[
"[",
"N"
],
[
"%",
"5"
],
[
"%1",
"5"
],
[
"%1",
"6"
],
[
"%1",
"7"
],
[
"2",
"]"
],
[
"H",
"2]"
],
[
"[C",
"H2]"
],
[
"%1",
"8"
],
[
"%1",
"9"
],
[
"%2",
"0"
],
[
"[",
"c"
],
[
"%2",
"1"
],
[
"[c",
"]"
],
[
"%2",
"2"
],
[
"%2",
"3"
],
[
"%",
"6"
],
[
"%2",
"4"
],
[
"[",
"1"
],
[
"%2",
"5"
],
[
"%2",
"6"
],
[
"+",
"]"
],
[
"%2",
"7"
],
[
"[",
"p"
],
[
"[p",
"H]"
],
[
"[N",
"]"
],
[
"%2",
"8"
],
[
"%2",
"9"
],
[
"[1",
"8"
],
[
"%3",
"0"
],
[
"F",
"]"
],
[
"%3",
"1"
],
[
"[18",
"F]"
],
[
"%3",
"2"
],
[
"[N",
"+]"
],
[
"%3",
"3"
],
[
"%3",
"4"
],
[
"%3",
"5"
],
[
"%3",
"6"
],
[
"%3",
"7"
],
[
"%3",
"8"
],
[
"%3",
"9"
],
[
"%",
"7"
],
[
"%4",
"0"
],
[
"%4",
"1"
],
[
"@",
"]"
],
[
"%4",
"2"
],
[
"%4",
"3"
],
[
"O",
"]"
],
[
"%4",
"4"
],
[
"%4",
"5"
],
[
"[",
"O]"
],
[
"%4",
"6"
],
[
"%4",
"7"
],
[
"%4",
"8"
],
[
"[",
"S"
],
[
"[N",
"H]"
],
[
"%4",
"9"
],
[
"%5",
"0"
],
[
"%",
"8"
],
[
"%5",
"1"
],
[
"@",
"@]"
],
[
"%5",
"2"
],
[
"%5",
"3"
],
[
"%5",
"4"
],
[
"[n",
"+]"
],
[
"%5",
"5"
],
[
"%5",
"6"
],
[
"%5",
"7"
],
[
"%5",
"8"
],
[
"%5",
"9"
],
[
"%6",
"0"
],
[
"%",
"9"
],
[
"[S",
"@@]"
],
[
"%6",
"1"
],
[
"%6",
"2"
],
[
"%6",
"3"
],
[
"[N",
"@@]"
],
[
"%6",
"4"
],
[
"%6",
"5"
],
[
"%6",
"6"
],
[
"[N",
"@]"
],
[
"[1",
"5"
],
[
"[S",
"@]"
],
[
"[15",
"n"
],
[
"%6",
"7"
],
[
"[",
"H"
],
[
"%6",
"8"
],
[
"[H",
"H]"
],
[
"H",
"+]"
],
[
"-",
"]"
],
[
"%6",
"9"
],
[
"%7",
"0"
],
[
"[15n",
"]"
],
[
"%7",
"1"
],
[
"%7",
"2"
],
[
"%7",
"3"
],
[
"%7",
"4"
],
[
"%7",
"5"
],
[
"%7",
"6"
],
[
"[S",
"]"
],
[
"%7",
"7"
],
[
"%7",
"8"
],
[
"[15n",
"H]"
],
[
"[n",
"H+]"
],
[
"%7",
"9"
],
[
"[18",
"O]"
],
[
"O",
"H]"
],
[
"%8",
"0"
],
[
"%8",
"1"
],
[
"[N",
"H+]"
],
[
"%8",
"2"
],
[
"%8",
"3"
],
[
"[1",
"2"
],
[
"[N",
"-]"
],
[
"%8",
"4"
],
[
"[18",
"OH]"
],
[
"%8",
"5"
],
[
"Y",
"]"
],
[
"[",
"s"
],
[
"[",
"Y]"
],
[
"[c",
"-]"
],
[
"%8",
"6"
],
[
"[s",
"i"
],
[
"%8",
"9"
],
[
"%8",
"8"
],
[
"%9",
"0"
],
[
"%8",
"7"
],
[
"[12",
"C"
],
[
"%9",
"1"
],
[
"-",
"2]"
],
[
"[1",
"H]"
],
[
"%9",
"2"
],
[
"[1",
"7"
],
[
"[1",
"9"
],
[
"[S",
"-2]"
],
[
"[si",
"+]"
],
[
"[19",
"F]"
],
[
"%9",
"3"
],
[
"3",
"]"
],
[
"[",
"3"
],
[
"H",
"3]"
],
[
"%9",
"4"
],
[
"[n",
"-]"
],
[
"%9",
"5"
],
[
"%9",
"6"
],
[
"[12",
"c"
],
[
"%9",
"7"
],
[
"%9",
"8"
],
[
"%9",
"9"
],
[
"[S",
"i"
],
[
"[17",
"O]"
],
[
"[Si",
"]"
],
[
"[12c",
"H]"
],
[
"Cl",
"]"
],
[
"N",
"H2]"
],
[
"[si",
"]"
],
[
"[1",
"4"
],
[
"[1",
"6"
],
[
"[12C",
"H2]"
],
[
"[17",
"OH]"
],
[
"[3",
"5"
],
[
"[12C",
"H3]"
],
[
"[35",
"Cl]"
],
[
"[S",
"H]"
],
[
"0",
"C"
],
[
"N",
"]"
],
[
"O",
"-]"
],
[
"[",
"O-]"
],
[
"[12C",
"]"
],
[
"[C",
"H3]"
],
[
"[1",
"3"
],
[
"+",
"2]"
],
[
"4",
"]"
],
[
"H",
"4]"
],
[
"[",
"2"
],
[
"[",
"B"
],
[
"C",
"H2]"
],
[
"F",
"H]"
],
[
"S",
"H2]"
],
[
"[",
"9"
],
[
"[3",
"4"
],
[
"[12c",
"]"
],
[
"[14",
"N]"
],
[
"[16",
"NH2]"
],
[
"[13",
"NH2]"
],
[
"[B",
"]"
],
[
"S",
"]"
],
[
"[",
"8"
],
[
"c",
"]"
],
[
"[1",
"0C"
],
[
"[S",
"H2]"
],
[
"[H",
"+]"
],
[
"[12C",
"H]"
],
[
"@",
"H]"
],
[
"C",
"]"
],
[
"C",
"H3]"
],
[
"C",
"H4]"
],
[
"N",
"H]"
],
[
"c",
"H]"
],
[
"[18",
"FH]"
],
[
"[2",
"0C"
],
[
"C",
"H]"
],
[
"Z",
"n"
],
[
"[",
"Zn"
],
[
"n",
"]"
],
[
"n",
"H]"
],
[
"o",
"]"
],
[
"s",
"]"
],
[
"[C",
"-]"
],
[
"[N",
"-2]"
],
[
"[1",
"1"
],
[
"[1",
"C]"
],
[
"[18",
"o]"
],
[
"[S",
"@"
],
[
"[si",
"H+]"
],
[
"[3",
"7"
],
[
"[Si",
"H3]"
],
[
"[14",
"NH]"
],
[
"[34",
"S]"
],
[
"[Zn",
"+2]"
],
[
"2",
"CH4]"
],
[
"@",
"@H]"
],
[
"H",
"H]"
],
[
"T",
"B"
],
[
"[C",
"+2]"
],
[
"[N",
"H"
],
[
"[N",
"+2]"
],
[
"[1",
"HH]"
],
[
"[18",
"c]"
],
[
"[18",
"n]"
],
[
"[15",
"CH2]"
],
[
"[si",
"H]"
],
[
"[12C",
"H4]"
],
[
"[12C",
"@H]"
],
[
"[12C",
"@@H]"
],
[
"[3",
"2"
],
[
"[3",
"3"
],
[
"[3",
"6"
],
[
"[3",
"8"
],
[
"[14",
"NH2]"
],
[
"[16",
"OH]"
],
[
"[16",
"N]"
],
[
"[16",
"nH]"
],
[
"[35",
"s]"
],
[
"[2",
"2CH4]"
],
[
"[9",
"CH2]"
],
[
"[9",
"c]"
],
[
"[8",
"CH2]"
],
[
"[8",
"CH]"
],
[
"[10C",
"H2]"
],
[
"[20C",
"]"
],
[
"[20C",
"H3]"
],
[
"[S@",
"TB"
],
[
"[37",
"Cl]"
],
[
"0",
"cH]"
],
[
"1",
"]"
],
[
"1",
"CH3]"
],
[
"2",
"+]"
],
[
"3",
"CH3]"
],
[
"4",
"CH3]"
],
[
"5",
"]"
],
[
"F",
"e"
],
[
"P",
"H]"
],
[
"P",
"1]"
],
[
"S",
"-2]"
],
[
"S",
"P1]"
],
[
"[",
"H]"
],
[
"[",
"F]"
],
[
"[",
"Fe"
],
[
"[",
"PH]"
],
[
"l",
"-]"
],
[
"[C",
"l-]"
],
[
"[1",
"H+]"
],
[
"[1",
"CH3]"
],
[
"[15",
"OH]"
],
[
"[15",
"c]"
],
[
"[15",
"CH4]"
],
[
"[15",
"cH]"
],
[
"[15",
"CH]"
],
[
"[si",
"-]"
],
[
"[17",
"F]"
],
[
"[17",
"FH]"
],
[
"[19",
"FH]"
],
[
"[3",
"C]"
],
[
"[14",
"n]"
],
[
"[14",
"nH]"
],
[
"[16",
"NH]"
],
[
"[13",
"O]"
],
[
"[13",
"N]"
],
[
"[2",
"0cH]"
],
[
"[B",
"-]"
],
[
"[9",
"CH4]"
],
[
"[9",
"cH]"
],
[
"[34",
"SH2]"
],
[
"[34",
"s]"
],
[
"[34",
"S-2]"
],
[
"[8",
"cH]"
],
[
"[10C",
"]"
],
[
"[10C",
"H3]"
],
[
"[10C",
"H4]"
],
[
"[11",
"1CH3]"
],
[
"[11",
"3CH3]"
],
[
"[11",
"4CH3]"
],
[
"[S@",
"SP1]"
],
[
"[37",
"SH2]"
],
[
"[NH",
"-]"
],
[
"[NH",
"2+]"
],
[
"[32",
"SH2]"
],
[
"[32",
"S]"
],
[
"[33",
"SH2]"
],
[
"[33",
"S]"
],
[
"[36",
"Cl]"
],
[
"[36",
"SH2]"
],
[
"[38",
"Cl]"
],
[
"[38",
"SH2]"
],
[
"[S@TB",
"3]"
],
[
"[S@TB",
"5]"
],
[
"[Fe",
"]"
]
]
}
}