Upload folder using huggingface_hub

special_tokens_map.json

@@ -1,7 +1,37 @@
 {
-  "bos_token": "@",
-  "eos_token": "*",
-  "mask_token": "#",
-  "pad_token": "!",
-  "sep_token": "<SEP>"
+  "bos_token": {
+    "content": "@",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "*",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "#",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "!",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "/",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer_config.json

@@ -1,16 +1,58 @@
 {
-  "tokenizer_class": "DNATokenizer",
-  "bos_token": "@",
-  "eos_token": "*",
-  "mask_token": "#",
-  "pad_token": "!",
-  "sep_token": "<SEP>",
-  "model_max_length": 2048,
-  "clean_up_tokenization_spaces": true,
+  "added_tokens_decoder": {
+    "6": {
+      "content": "*",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "#",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "@",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "!",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "/",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
   "auto_map": {
     "AutoTokenizer": [
       "tokenizers.DNATokenizer",
       null
     ]
-  }
+  },
+  "bos_token": "@",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "*",
+  "mask_token": "#",
+  "model_max_length": 2048,
+  "pad_token": "!",
+  "sep_token": "/",
+  "tokenizer_class": "DNATokenizer"
 }

tokenizers.py

@@ -2,46 +2,51 @@ from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
 from typing import List, Optional, Union
 import os
 
-# Special tokens
 MASK = "#"
-PAD = "!"
-BOS = "@"
-EOS = "*"
+MSA_PAD = "!"
+UL_ALPHABET_PLUS = "GATCN-*#@!/[]{}"
+MSA_AAS = "GATCN-"
+GAP = "-"
+START = "@"
+STOP = "*"
 SEP = "/"
-
-# Generatable alphabet (only what the model should predict)
-BASE_ALPHABET = "ACGT/{}"   # 7 tokens
+END_AL = "]"
+END_UL = "}"
+START_AL = "["
+START_UL = "{"
 
 class DNATokenizer(PreTrainedTokenizer):
 
     def __init__(
         self,
-        dna_alphabet: str = BASE_ALPHABET,
+        dna_alphabet: str = UL_ALPHABET_PLUS,
         model_max_length: int = 2048,
-        pad_token=PAD,
+        pad_token=MSA_PAD,
         mask_token=MASK,
-        bos_token=BOS,
-        eos_token=EOS,
+        all_aas=MSA_AAS,
+        gap_token=GAP,
+        bos_token=START,
+        eos_token=STOP,
         sep_token=SEP,
         **kwargs
     ):
-        # Build vocabulary
-        self.alphabet = list(dna_alphabet)
+        """Character tokenizer for Hugging Face transformers.
+
+        model_max_length (int): Model maximum sequence length.
+        """
+        self.alphabet = list("".join(dna_alphabet))
+        self.all_aas = list("".join(all_aas))
         self.a_to_i = {u: i for i, u in enumerate(self.alphabet)}
         self.i_to_a = {i: u for i, u in enumerate(self.alphabet)}
+        self.gap_token = gap_token
 
-        # Wrap special tokens
-        # def ensure_added(token):
-        #     if isinstance(token, AddedToken):
-        #         return token
-        #     return AddedToken(token, lstrip=False, rstrip=False)
-
-        # bos_token = ensure_added(bos_token)
-        # eos_token = ensure_added(eos_token)
-        # sep_token = ensure_added(sep_token)
-        # mask_token = ensure_added(mask_token)
-        # pad_token = ensure_added(pad_token)
-
+        
+        bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
+        eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
+        sep_token = AddedToken(sep_token, lstrip=False, rstrip=False) if isinstance(sep_token, str) else sep_token
+        mask_token = AddedToken(mask_token, lstrip=False, rstrip=False) if isinstance(mask_token, str) else mask_token 
+        pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
+        gap_token = AddedToken(gap_token, lstrip=False, rstrip=False) if isinstance(gap_token, str) else gap_token
 
         super().__init__(
             pad_token=pad_token,
@@ -56,6 +61,10 @@ class DNATokenizer(PreTrainedTokenizer):
     @property
     def vocab_size(self):
         return len(self.alphabet)
+    
+    @property
+    def gap_token_id(self):
+        return self.convert_tokens_to_ids(self.gap_token)
 
     def get_vocab(self):
         return self.a_to_i
@@ -73,9 +82,10 @@ class DNATokenizer(PreTrainedTokenizer):
         return "".join(tokens)
 
     def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        result = token_ids_0
         if token_ids_1 is not None:
             raise NotImplementedError("This tokenizer does not support two sequences")
-        return token_ids_0
+        return result
 
     def get_special_tokens_mask(
         self,
@@ -89,15 +99,28 @@ class DNATokenizer(PreTrainedTokenizer):
                 token_ids_1=token_ids_1,
                 already_has_special_tokens=True,
             )
-        return [0] * len(token_ids_0)
+
+        result = [0] * len(token_ids_0)
+        if token_ids_1 is not None:
+            raise NotImplementedError("This tokenizer does not support two sequences")
+
+        return result
 
     def create_token_type_ids_from_sequences(
         self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
     ) -> List[int]:
+        """
+        Identifies the type of token. 0 for the first sentence, 1 for the second sentence if it exists
+        """
+
+        result = len(token_ids_0) * [0]
+
         if token_ids_1 is not None:
             raise NotImplementedError("This tokenizer does not support two sequences")
-        return [0] * len(token_ids_0)
+        return result
 
+    def save_pretrained(self, save_directory: Union[str, os.PathLike], **kwargs):
+        super().save_pretrained(save_directory, **kwargs)
+    
     def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None):
-        # No separate vocab file needed
         return ()