SchwallerGroup
/

MechSMILES_tokenizer

Transformers

Model card Files Files and versions

xet

Community

Volowan commited on Sep 19, 2025

Commit

e8318f3

verified ·

1 Parent(s): 69aff18

Upload tokenizer

Browse files

Files changed (2) hide show

tokenizer.json +133 -132
tokenizer_config.json +5 -5

tokenizer.json CHANGED Viewed

@@ -13,7 +13,7 @@
       "special": true
     },
     {
-      "id": 254,
       "content": "[bos]",
       "single_word": false,
       "lstrip": false,
@@ -22,7 +22,7 @@
       "special": true
     },
     {
-      "id": 255,
       "content": "[pad]",
       "single_word": false,
       "lstrip": false,
@@ -31,7 +31,7 @@
       "special": true
     },
     {
-      "id": 256,
       "content": "[eos]",
       "single_word": false,
       "lstrip": false,
@@ -40,7 +40,7 @@
       "special": true
     },
     {
-      "id": 257,
       "content": "[sep]",
       "single_word": false,
       "lstrip": false,
@@ -49,7 +49,7 @@
       "special": true
     },
     {
-      "id": 258,
       "content": "[unk]",
       "single_word": false,
       "lstrip": false,
@@ -212,133 +212,134 @@
       "H": 129,
       "H2": 130,
       "H3": 131,
-      "[": 132,
-      "]": 133,
-      ":": 134,
-      "=": 135,
-      "#": 136,
-      "$": 137,
-      "\\": 138,
-      "/": 139,
-      "(": 140,
-      ")": 141,
-      ".": 142,
-      "[prod]": 143,
-      "[reac]": 144,
-      "[mech]": 145,
-      "+": 146,
-      "-": 147,
-      "@": 148,
-      "*": 149,
-      "0": 150,
-      "1": 151,
-      "2": 152,
-      "3": 153,
-      "4": 154,
-      "5": 155,
-      "6": 156,
-      "7": 157,
-      "8": 158,
-      "9": 159,
-      "10": 160,
-      "11": 161,
-      "12": 162,
-      "13": 163,
-      "14": 164,
-      "15": 165,
-      "16": 166,
-      "17": 167,
-      "18": 168,
-      "19": 169,
-      "20": 170,
-      "21": 171,
-      "22": 172,
-      "23": 173,
-      "24": 174,
-      "25": 175,
-      "26": 176,
-      "27": 177,
-      "28": 178,
-      "29": 179,
-      "30": 180,
-      "31": 181,
-      "32": 182,
-      "33": 183,
-      "34": 184,
-      "35": 185,
-      "36": 186,
-      "37": 187,
-      "38": 188,
-      "39": 189,
-      "40": 190,
-      "41": 191,
-      "42": 192,
-      "43": 193,
-      "44": 194,
-      "45": 195,
-      "46": 196,
-      "47": 197,
-      "48": 198,
-      "49": 199,
-      "50": 200,
-      "51": 201,
-      "52": 202,
-      "53": 203,
-      "54": 204,
-      "55": 205,
-      "56": 206,
-      "57": 207,
-      "58": 208,
-      "59": 209,
-      "60": 210,
-      "61": 211,
-      "62": 212,
-      "63": 213,
-      "64": 214,
-      "65": 215,
-      "66": 216,
-      "67": 217,
-      "68": 218,
-      "69": 219,
-      "70": 220,
-      "71": 221,
-      "72": 222,
-      "73": 223,
-      "74": 224,
-      "75": 225,
-      "76": 226,
-      "77": 227,
-      "78": 228,
-      "79": 229,
-      "80": 230,
-      "81": 231,
-      "82": 232,
-      "83": 233,
-      "84": 234,
-      "85": 235,
-      "86": 236,
-      "87": 237,
-      "88": 238,
-      "89": 239,
-      "90": 240,
-      "91": 241,
-      "92": 242,
-      "93": 243,
-      "94": 244,
-      "95": 245,
-      "96": 246,
-      "97": 247,
-      "98": 248,
-      "99": 249,
-      "%": 250,
-      ",": 251,
-      ";": 252,
-      "|": 253,
-      "[bos]": 254,
-      "[pad]": 255,
-      "[eos]": 256,
-      "[sep]": 257,
-      "[unk]": 258
     },
     "unk_token": "[unk]"
   }

       "special": true
     },
     {
+      "id": 255,
       "content": "[bos]",
       "single_word": false,
       "lstrip": false,
       "special": true
     },
     {
+      "id": 256,
       "content": "[pad]",
       "single_word": false,
       "lstrip": false,
       "special": true
     },
     {
+      "id": 257,
       "content": "[eos]",
       "single_word": false,
       "lstrip": false,
       "special": true
     },
     {
+      "id": 258,
       "content": "[sep]",
       "single_word": false,
       "lstrip": false,
       "special": true
     },
     {
+      "id": 259,
       "content": "[unk]",
       "single_word": false,
       "lstrip": false,
       "H": 129,
       "H2": 130,
       "H3": 131,
+      "H4": 132,
+      "[": 133,
+      "]": 134,
+      ":": 135,
+      "=": 136,
+      "#": 137,
+      "$": 138,
+      "\\": 139,
+      "/": 140,
+      "(": 141,
+      ")": 142,
+      ".": 143,
+      "[prod]": 144,
+      "[reac]": 145,
+      "[mech]": 146,
+      "+": 147,
+      "-": 148,
+      "@": 149,
+      "*": 150,
+      "0": 151,
+      "1": 152,
+      "2": 153,
+      "3": 154,
+      "4": 155,
+      "5": 156,
+      "6": 157,
+      "7": 158,
+      "8": 159,
+      "9": 160,
+      "10": 161,
+      "11": 162,
+      "12": 163,
+      "13": 164,
+      "14": 165,
+      "15": 166,
+      "16": 167,
+      "17": 168,
+      "18": 169,
+      "19": 170,
+      "20": 171,
+      "21": 172,
+      "22": 173,
+      "23": 174,
+      "24": 175,
+      "25": 176,
+      "26": 177,
+      "27": 178,
+      "28": 179,
+      "29": 180,
+      "30": 181,
+      "31": 182,
+      "32": 183,
+      "33": 184,
+      "34": 185,
+      "35": 186,
+      "36": 187,
+      "37": 188,
+      "38": 189,
+      "39": 190,
+      "40": 191,
+      "41": 192,
+      "42": 193,
+      "43": 194,
+      "44": 195,
+      "45": 196,
+      "46": 197,
+      "47": 198,
+      "48": 199,
+      "49": 200,
+      "50": 201,
+      "51": 202,
+      "52": 203,
+      "53": 204,
+      "54": 205,
+      "55": 206,
+      "56": 207,
+      "57": 208,
+      "58": 209,
+      "59": 210,
+      "60": 211,
+      "61": 212,
+      "62": 213,
+      "63": 214,
+      "64": 215,
+      "65": 216,
+      "66": 217,
+      "67": 218,
+      "68": 219,
+      "69": 220,
+      "70": 221,
+      "71": 222,
+      "72": 223,
+      "73": 224,
+      "74": 225,
+      "75": 226,
+      "76": 227,
+      "77": 228,
+      "78": 229,
+      "79": 230,
+      "80": 231,
+      "81": 232,
+      "82": 233,
+      "83": 234,
+      "84": 235,
+      "85": 236,
+      "86": 237,
+      "87": 238,
+      "88": 239,
+      "89": 240,
+      "90": 241,
+      "91": 242,
+      "92": 243,
+      "93": 244,
+      "94": 245,
+      "95": 246,
+      "96": 247,
+      "97": 248,
+      "98": 249,
+      "99": 250,
+      "%": 251,
+      ",": 252,
+      ";": 253,
+      "|": 254,
+      "[bos]": 255,
+      "[pad]": 256,
+      "[eos]": 257,
+      "[sep]": 258,
+      "[unk]": 259
     },
     "unk_token": "[unk]"
   }

tokenizer_config.json CHANGED Viewed

@@ -8,7 +8,7 @@
       "single_word": false,
       "special": true
     },
-    "254": {
       "content": "[bos]",
       "lstrip": false,
       "normalized": false,
@@ -16,7 +16,7 @@
       "single_word": false,
       "special": true
     },
-    "255": {
       "content": "[pad]",
       "lstrip": false,
       "normalized": false,
@@ -24,7 +24,7 @@
       "single_word": false,
       "special": true
     },
-    "256": {
       "content": "[eos]",
       "lstrip": false,
       "normalized": false,
@@ -32,7 +32,7 @@
       "single_word": false,
       "special": true
     },
-    "257": {
       "content": "[sep]",
       "lstrip": false,
       "normalized": false,
@@ -40,7 +40,7 @@
       "single_word": false,
       "special": true
     },
-    "258": {
       "content": "[unk]",
       "lstrip": false,
       "normalized": false,

       "single_word": false,
       "special": true
     },
+    "255": {
       "content": "[bos]",
       "lstrip": false,
       "normalized": false,
       "single_word": false,
       "special": true
     },
+    "256": {
       "content": "[pad]",
       "lstrip": false,
       "normalized": false,
       "single_word": false,
       "special": true
     },
+    "257": {
       "content": "[eos]",
       "lstrip": false,
       "normalized": false,
       "single_word": false,
       "special": true
     },
+    "258": {
       "content": "[sep]",
       "lstrip": false,
       "normalized": false,
       "single_word": false,
       "special": true
     },
+    "259": {
       "content": "[unk]",
       "lstrip": false,
       "normalized": false,