Upload tokenizer

tokenizer.py

@@ -4,22 +4,7 @@ from typing import List, Optional, Union
 
 import numpy as np
 import torch
-from transformers import PretrainedConfig, PreTrainedTokenizer
-
-
-class BinnedOmicTokenizerConfig(PretrainedConfig):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self.n_expressions_bins = kwargs.get("n_expressions_bins", 64)
-        self.min_omic_value = kwargs.get("min_omic_value", 0.0)
-        self.max_omic_value = kwargs.get("max_omic_value", 1.0)
-        self.use_max_normalization = kwargs.get("use_max_normalization", True)
-        self.normalization_factor = kwargs.get(
-            "normalization_factor", 5.547176906585117
-        )
-        self.prepend_cls_token = kwargs.get("prepend_cls_token", False)
-        self.fixed_sequence_length = kwargs.get("fixed_sequence_length", None)
-        self.unpadded_length = kwargs.get("unpadded_length", None)
+from transformers import PreTrainedTokenizer
 
 
 class BinnedOmicTokenizer(PreTrainedTokenizer):
@@ -77,6 +62,9 @@ class BinnedOmicTokenizer(PreTrainedTokenizer):
     def _tokenize(self, text, **kwargs):
         raise NotImplementedError("Use `encode` or `batch_encode_plus` methods.")
 
+    def decode(self, token_ids, **kwargs):
+        return [self._convert_id_to_token(i) for i in token_ids]
+
     def encode(
         self,
         gene_expr: Union[np.ndarray, List[float]],

tokenizer_config.json

@@ -1,17 +1,12 @@
 {
-  "tokenizer_class": "BinnedOmicTokenizer",
-  "n_expressions_bins": 64,
-  "min_omic_value": 0.0,
-  "max_omic_value": 1.0,
-  "use_max_normalization": true,
-  "normalization_factor": 5.547176906585117,
-  "prepend_cls_token": false,
-  "fixed_sequence_length": null,
-  "unpadded_length": null,
+  "added_tokens_decoder": {},
   "auto_map": {
     "AutoTokenizer": [
       "tokenizer.BinnedOmicTokenizer",
       null
     ]
-  }
-}
+  },
+  "clean_up_tokenization_spaces": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "BinnedOmicTokenizer"
+}