Upload tokenizer

special_tokens_map.json

@@ -1,8 +1,5 @@
 {
-  "bos_token": "<bos>",
   "cls_token": "<cls>",
-  "eos_token": "<eos>",
   "mask_token": "<mask>",
-  "pad_token": "<pad>",
-  "unk_token": "<unk>"
+  "pad_token": "<pad>"
 }

tokenizer.py

@@ -21,7 +21,7 @@ class BinnedOmicTokenizer(PreTrainedTokenizer):
         **kwargs,
     ):
         bin_tokens = [str(i) for i in range(n_expressions_bins)]
-        special_tokens = ["<pad>", "<mask>", "<unk>", "<bos>", "<eos>", "<cls>"]
+        special_tokens = ["<pad>", "<mask>", "<cls>"]
 
         vocab = {tok: i for i, tok in enumerate(bin_tokens)}
         offset = len(vocab)
@@ -47,17 +47,11 @@ class BinnedOmicTokenizer(PreTrainedTokenizer):
 
         self.pad_token = "<pad>"
         self.mask_token = "<mask>"
-        self.unk_token = "<unk>"
-        self.bos_token = "<bos>"
-        self.eos_token = "<eos>"
         self.cls_token = "<cls>"
 
         super().__init__(
             pad_token=self.pad_token,
             mask_token=self.mask_token,
-            unk_token=self.unk_token,
-            bos_token=self.bos_token,
-            eos_token=self.eos_token,
             cls_token=self.cls_token,
             **kwargs,
         )

tokenizer_config.json

@@ -1,11 +1,40 @@
 {
-  "tokenizer_class": "BinnedOmicTokenizer",
-  "n_expressions_bins": 64,
-  "min_omic_value": 0.0,
-  "max_omic_value": 1.0,
-  "use_max_normalization": true,
-  "normalization_factor": 5.547176906585117,
-  "prepend_cls_token": false,
-  "fixed_sequence_length": null,
-  "unpadded_length": null
-}
+  "added_tokens_decoder": {
+    "64": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "66": {
+      "content": "<cls>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenizer.BinnedOmicTokenizer",
+      null
+    ]
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<cls>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "BinnedOmicTokenizer"
+}

vocab.json

@@ -1 +1 @@
-{"0": 0, "1": 1, "2": 2, "3": 3, "4": 4, "5": 5, "6": 6, "7": 7, "8": 8, "9": 9, "10": 10, "11": 11, "12": 12, "13": 13, "14": 14, "15": 15, "16": 16, "17": 17, "18": 18, "19": 19, "20": 20, "21": 21, "22": 22, "23": 23, "24": 24, "25": 25, "26": 26, "27": 27, "28": 28, "29": 29, "30": 30, "31": 31, "32": 32, "33": 33, "34": 34, "35": 35, "36": 36, "37": 37, "38": 38, "39": 39, "40": 40, "41": 41, "42": 42, "43": 43, "44": 44, "45": 45, "46": 46, "47": 47, "48": 48, "49": 49, "50": 50, "51": 51, "52": 52, "53": 53, "54": 54, "55": 55, "56": 56, "57": 57, "58": 58, "59": 59, "60": 60, "61": 61, "62": 62, "63": 63, "<pad>": 64, "<mask>": 65, "<unk>": 66, "<bos>": 67, "<eos>": 68, "<cls>": 69}
+{"0": 0, "1": 1, "2": 2, "3": 3, "4": 4, "5": 5, "6": 6, "7": 7, "8": 8, "9": 9, "10": 10, "11": 11, "12": 12, "13": 13, "14": 14, "15": 15, "16": 16, "17": 17, "18": 18, "19": 19, "20": 20, "21": 21, "22": 22, "23": 23, "24": 24, "25": 25, "26": 26, "27": 27, "28": 28, "29": 29, "30": 30, "31": 31, "32": 32, "33": 33, "34": 34, "35": 35, "36": 36, "37": 37, "38": 38, "39": 39, "40": 40, "41": 41, "42": 42, "43": 43, "44": 44, "45": 45, "46": 46, "47": 47, "48": 48, "49": 49, "50": 50, "51": 51, "52": 52, "53": 53, "54": 54, "55": 55, "56": 56, "57": 57, "58": 58, "59": 59, "60": 60, "61": 61, "62": 62, "63": 63, "<pad>": 64, "<mask>": 65, "<cls>": 66}