Adapt tokenization_interns1.py to transformers>=5.0.0

tokenization_interns1.py

@@ -25,24 +25,27 @@ import regex as re
 import sentencepiece as spm
 from collections import OrderedDict
 
-from transformers.tokenization_utils import PreTrainedTokenizer
 from transformers.tokenization_utils_base import AddedToken, TextInput
-from transformers.models.qwen2.tokenization_qwen2 import Qwen2Tokenizer
 from transformers.utils import logging
+import transformers
+from packaging import version
+if version.parse(transformers.__version__) >= version.parse("5.0.0"):
+    from transformers.tokenization_python import PreTrainedTokenizer
+else:
+    from transformers.tokenization_utils import PreTrainedTokenizer
 
 
 logger = logging.get_logger(__name__)
 
 try:
-    from rdkit import Chem
-    from rdkit import RDLogger
+    from rdkit import Chem, RDLogger
 
     RDLogger.DisableLog("rdApp.error")
     RDLogger.DisableLog("rdApp.*")
     RDKIT_AVAILABLE = True
 except ImportError:
     logger.warning_once(
-        f"If tokenization with SMILES formula is of necessity, please 'pip install RDKit' for better tokenization quality."
+        "If tokenization with SMILES formula is of necessity, please 'pip install RDKit' for better tokenization quality."
     )
     RDKIT_AVAILABLE = False
 
@@ -60,7 +63,6 @@ PRETOKENIZE_REGEX = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p
 class InternS1CheckModuleMixin(ABC):
     """
     Basic auto-detection module.
-
     Note that short strings are ignored by this module.
     """
     def __init__(self, *, min_length: int):
@@ -122,7 +124,6 @@ class InternS1CheckModuleMixin(ABC):
 class FastaCheckModule(InternS1CheckModuleMixin):
     """
     Protein sequence auto-detection module.
-
     Automatically detects protein sequence using regex patterns.
     """
     def __init__(self, *, min_length: int = 27):
@@ -160,7 +161,6 @@ elements = [
 class SmilesCheckModule(InternS1CheckModuleMixin):
     """
     SMILES molecular sequence auto-detection module.
-
     Automatically detects and validates SMILES strings in text using regex patterns
     or chemical syntax rules. Uses RDKit for precise validation when available,
     otherwise falls back to rule-based validation.
@@ -343,28 +343,61 @@ class SmilesCheckModule(InternS1CheckModuleMixin):
         return self.check_brackets(text)
 
 
-class InternS1Tokenizer(Qwen2Tokenizer):
+@lru_cache
+# Copied from transformers.models.gpt2.tokenization_gpt2.bytes_to_unicode
+def bytes_to_unicode():
     """
-    Construct an InternS1 tokenizer. Based on byte-level Byte-Pair-Encoding.
+    Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
+    characters the bpe code barfs on.
+    The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
+    if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
+    decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
+    tables between utf-8 bytes and unicode strings.
+    """
+    bs = (
+        list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
+    )
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8 + n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+
+# Copied from transformers.models.gpt2.tokenization_gpt2.get_pairs
+def get_pairs(word):
+    """
+    Return set of symbol pairs in a word.
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
 
+class InternS1Tokenizer(PreTrainedTokenizer):
+    """
+    Construct an InternS1 tokenizer. Based on byte-level Byte-Pair-Encoding.
     Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
     be encoded differently whether it is at the beginning of the sentence (without space) or not:
-
     ```python
     >>> from transformers import AutoTokenizer
-
     >>> tokenizer = AutoTokenizer.from_pretrained("InternS1Tokenizer", trust_remote_code=True)
     >>> tokenizer("Hello world")["input_ids"]
     [9707, 1879]
-
     >>> tokenizer(" Hello world")["input_ids"]
     [21927, 1879]
     ```
     This is expected.
-
     Include custom extension to support better domain-specific text tokenization, leveraging a separately trained tokenizer model.
     Users should refer to this superclass [`PreTrainedTokenizer`] for more information regarding those overloaded methods
-
     Args:
         vocab_file (`str`):
             Path to the vocabulary file.
@@ -408,6 +441,54 @@ class InternS1Tokenizer(Qwen2Tokenizer):
         split_special_tokens=False,
         **kwargs,
     ):
+        bos_token = (
+            AddedToken(bos_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(bos_token, str)
+            else bos_token
+        )
+        eos_token = (
+            AddedToken(eos_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(eos_token, str)
+            else eos_token
+        )
+        unk_token = (
+            AddedToken(unk_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(unk_token, str)
+            else unk_token
+        )
+        pad_token = (
+            AddedToken(pad_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(pad_token, str)
+            else pad_token
+        )
+
+        with open(vocab_file, encoding="utf-8") as vocab_handle:
+            self.encoder = json.load(vocab_handle)
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.errors = errors  # how to handle errors in decoding
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        bpe_merges = []
+        with open(merges_file, encoding="utf-8") as merges_handle:
+            for i, line in enumerate(merges_handle):
+                line = line.strip()
+                if (i == 0 and line.startswith("#version:")) or not line:
+                    continue
+                bpe_merges.append(tuple(line.split()))
+        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
+        # NOTE: the cache can grow without bound and will get really large for long running processes
+        # (esp. for texts of language that do not use space between word, e.g. Chinese); technically
+        # not a memory leak but appears as one.
+        # GPT2Tokenizer has the same problem, so let's be consistent.
+        self.cache = {}
+
+        self.pat = re.compile(PRETOKENIZE_REGEX)
+
+        if kwargs.get("add_prefix_space", False):
+            logger.warning_once(
+                f"{self.__class__.__name} does not support `add_prefix_space`, setting it to True has no effect."
+            )
+
         self.extra_tokenizer_start_mapping = {}
         self.extra_tokenizer_end_mapping = {}
         self._extra_special_tokens = []
@@ -460,6 +541,7 @@ class InternS1Tokenizer(Qwen2Tokenizer):
             pad_token=pad_token,
             clean_up_tokenization_spaces=clean_up_tokenization_spaces,
             split_special_tokens=split_special_tokens,
+            special_tokens_pattern="none",
             **kwargs,
         )
 
@@ -497,6 +579,10 @@ class InternS1Tokenizer(Qwen2Tokenizer):
         """Overload method"""
         return self.vocab_size
 
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.get_vocab
+    def get_vocab(self):
+        return dict(self.encoder, **self.added_tokens_encoder)
+
     @property
     def logical_auto_tokens(self):
         """Tokens that won't be decoded and only for switching tokenizer"""
@@ -623,9 +709,7 @@ class InternS1Tokenizer(Qwen2Tokenizer):
     def tokenize(self, text: TextInput, **kwargs) -> List[str]:
         """
         Converts a string into a sequence of tokens, using the tokenizer.
-
         It will switch to domain-specific tokenizer once encountering extra/logical sp tokens.
-
         Args:
             text: TextInput
         """
@@ -633,9 +717,6 @@ class InternS1Tokenizer(Qwen2Tokenizer):
 
         text, kwargs = self.prepare_for_tokenization(text, **kwargs)
 
-        if kwargs:
-            logger.warning(f"Keyword arguments {kwargs} not recognized.")
-
         if hasattr(self, "do_lower_case") and self.do_lower_case:
             # convert non-special tokens to lowercase. Might be super slow as well?
             escaped_special_toks = [re.escape(s_tok) for s_tok in (self.all_special_tokens)]
@@ -738,7 +819,6 @@ class InternS1Tokenizer(Qwen2Tokenizer):
     def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int:
         """
         Modified from `transformers.tokenization_utils._add_tokens`.
-
         This adaptation supports dynamic tokenizer length due to supplementary tokenizers (e.g., domain-specific or scientific text tokenizers).
         """
         added_tokens = 0
@@ -785,6 +865,7 @@ class InternS1Tokenizer(Qwen2Tokenizer):
             self._added_tokens_encoder[token.content] = token_index
             if self.verbose:
                 logger.info(f"Adding {token} to the vocabulary")
+
         self._update_trie()
         self._update_total_vocab_size()
 
@@ -797,7 +878,6 @@ class InternS1Tokenizer(Qwen2Tokenizer):
     def _tokenize(self, text, **kwargs):
         """
         Modified from `transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._tokenize`.
-
         This adaptation supports domain-specific tokenizers.
         """
         extra_tokenizer_stack = kwargs.pop("extra_tokenizer_stack", False)
@@ -814,6 +894,49 @@ class InternS1Tokenizer(Qwen2Tokenizer):
         else:
             return self._bpe_tokenize(text)
 
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.bpe
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token)
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token
+
+        while True:
+            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                except ValueError:
+                    new_word.extend(word[i:])
+                    break
+                else:
+                    new_word.extend(word[i:j])
+                    i = j
+
+                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
+                    new_word.append(first + second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = " ".join(word)
+        self.cache[token] = word
+        return word
+
     def _bpe_tokenize(self, text, **kwargs):
         text = text.replace(
             "▁", " "
@@ -829,15 +952,11 @@ class InternS1Tokenizer(Qwen2Tokenizer):
     def convert_tokens_to_ids(self, tokens: Union[str, List[str]]) -> Union[int, List[int]]:
         """
         Modified from `transformers.tokenization_utils.PreTrainedTokenzier.convert_tokens_to_ids`.
-
         Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the
         vocabulary.
-
         This adaptation supports domain-specific tokenizers.
-
         Args:
             tokens (`str` or `List[str]`): One or several token(s) to convert to token id(s).
-
         Returns:
             `int` or `List[int]`: The token id or list of token ids.
         """
@@ -865,7 +984,6 @@ class InternS1Tokenizer(Qwen2Tokenizer):
     def _convert_token_to_id_with_added_voc(self, token, **kwargs):
         """
         Modified from `transformers.tokenization_utils.PreTrainedTokenzier._convert_token_to_id_with_added_voc`.
-
         This adaptation supports domain-specific tokenizers.
         """
         if token is None:
@@ -878,9 +996,7 @@ class InternS1Tokenizer(Qwen2Tokenizer):
     def _convert_token_to_id(self, token, **kwargs):
         """
         Modified from `transformers.tokenization_utils.PreTrainedTokenzier._convert_token_to_id`.
-
         Converts a token (str) in an id using the vocab.
-
         Fall back to original tokenizer once OOV.
         """
         extra_tokenizer_stack = kwargs.pop("extra_tokenizer_stack", False)
@@ -978,3 +1094,4 @@ class InternS1Tokenizer(Qwen2Tokenizer):
 
 
 __all__ = ["InternS1Tokenizer"]
+