Upload tokenizer

tokenization_decodon.py

@@ -27,6 +27,30 @@ class DeCodonTokenizer(PreTrainedTokenizer):
         else:
             return ["".join(codon) for codon in product("ACGU", repeat=3)]
 
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
+        """
+        Instantiate a DeCodonTokenizer from a pre-trained tokenizer.
+        """
+        # Handle the case where we're loading from a local directory
+        if os.path.isdir(pretrained_model_name_or_path):
+            vocab_file = os.path.join(pretrained_model_name_or_path, "vocab.json")
+            if os.path.exists(vocab_file):
+                kwargs["vocab_file"] = vocab_file
+        else:
+            # For hub loading, try to get the vocab file from the cached download
+            from transformers.utils import cached_file
+            try:
+                vocab_file = cached_file(pretrained_model_name_or_path, "vocab.json")
+                if vocab_file:
+                    kwargs["vocab_file"] = vocab_file
+            except Exception:
+                # If vocab.json is not found, continue without it (use default vocab)
+                pass
+        
+        # Create instance with the vocab_file parameter
+        return cls(*inputs, **kwargs)
+
     def __init__(
         self,
         vocab_file=None,
@@ -49,13 +73,17 @@ class DeCodonTokenizer(PreTrainedTokenizer):
             with open(vocab_file, "r") as f:
                 self.encoder = json.load(f)
                 self.decoder = {i: k for k, i in self.encoder.items()}
+            
+            self.compiled_regex = re.compile(
+                "|".join(list(self.encoder.keys()) + [r"\S"])
+            )
         else:
             self.encoder = {k: i for i, k in enumerate(self.special_tokens + self.codons)}
             self.decoder = {i: k for k, i in self.encoder.items()}
         
-        self.compiled_regex = re.compile(
-            "|".join(self.codons + self.special_tokens + [r"\S"])
-        )
+            self.compiled_regex = re.compile(
+                "|".join(self.codons + self.special_tokens + [r"\S"])
+            )
 
         super().__init__(
             cls_token=cls_token,