Transformers v5 compatibility fixes

config.json

@@ -4,6 +4,7 @@
  ],
  "attention_dropout": 0.0,
  "auto_map": {
+  "AutoConfig": "configuration_qwen.Qwen2Config",
   "AutoModel": "modeling_qwen.Qwen2Model",
   "AutoModelForCausalLM": "modeling_qwen.Qwen2ForCausalLM",
   "AutoModelForSequenceClassification": "modeling_qwen.Qwen2ForSequenceClassification"

configuration_qwen.py

@@ -0,0 +1,109 @@
+# coding=utf-8
+# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Qwen2 model configuration"""
+
+from transformers import PretrainedConfig
+
+
+class Qwen2Config(PretrainedConfig):
+    model_type = "qwen2"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    # Default tensor parallel plan for base model `Qwen2`
+    base_model_tp_plan = {
+        "layers.*.self_attn.q_proj": "colwise",
+        "layers.*.self_attn.k_proj": "colwise",
+        "layers.*.self_attn.v_proj": "colwise",
+        "layers.*.self_attn.o_proj": "rowwise",
+        "layers.*.mlp.gate_proj": "colwise",
+        "layers.*.mlp.up_proj": "colwise",
+        "layers.*.mlp.down_proj": "rowwise",
+    }
+    base_model_pp_plan = {
+        "embed_tokens": (["input_ids"], ["inputs_embeds"]),
+        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+        "norm": (["hidden_states"], ["hidden_states"]),
+    }
+
+    def __init__(
+        self,
+        vocab_size=151936,
+        hidden_size=4096,
+        intermediate_size=22016,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=32,
+        hidden_act="silu",
+        max_position_embeddings=32768,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        use_sliding_window=False,
+        sliding_window=4096,
+        max_window_layers=28,
+        layer_types=None,
+        attention_dropout=0.0,
+        pad_token_id: int | None = None,
+        bos_token_id: int | None = None,
+        eos_token_id: int | None = None,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.use_sliding_window = use_sliding_window
+        self.sliding_window = sliding_window if self.use_sliding_window else None
+        self.max_window_layers = max_window_layers
+
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.attention_dropout = attention_dropout
+        # Validate the correctness of rotary position embeddings parameters
+        # BC: if there is a 'type' field, move it to 'rope_type'.
+        if self.rope_scaling is not None and "type" in self.rope_scaling:
+            self.rope_scaling["rope_type"] = self.rope_scaling["type"]
+        
+        self.layer_types = layer_types
+        if self.layer_types is None:
+            self.layer_types = [
+                "sliding_attention"
+                if self.sliding_window is not None and i >= self.max_window_layers
+                else "full_attention"
+                for i in range(self.num_hidden_layers)
+            ]
+
+        self.pad_token_id = pad_token_id
+        self.bos_token_id = bos_token_id
+        self.eos_token_id = eos_token_id
+        self.tie_word_embeddings = tie_word_embeddings
+        super().__init__(**kwargs)
+
+
+__all__ = ["Qwen2Config"]

modeling_qwen.py

@@ -18,12 +18,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ PyTorch Qwen2 model."""
-from transformers import Qwen2Config
+from contextlib import nullcontext
+from transformers import Qwen2Config, ROPE_INIT_FUNCTIONS
 import inspect
 import math
-import os
 import warnings
-from typing import List, Optional, Tuple, Union
+from typing import List, Optional, Tuple, Union, Callable
 
 import torch
 import torch.nn.functional as F
@@ -44,8 +44,6 @@ from transformers.utils import (
     logging,
     replace_return_docstrings,
 )
-
-
 if is_flash_attn_2_available():
     from flash_attn import flash_attn_func, flash_attn_varlen_func
     from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input  # noqa
@@ -65,6 +63,19 @@ QWEN2_PRETRAINED_MODEL_ARCHIVE_LIST = [
 ]
 
 
+def maybe_autocast(
+    device_type: str,
+    dtype: Optional["_dtype"] = None,
+    enabled: bool = True,
+    cache_enabled: bool | None = None,
+):
+    if torch.is_autocast_enabled(device_type) or enabled:
+        return torch.autocast(device_type, dtype=dtype, enabled=enabled, cache_enabled=cache_enabled)
+    else:
+        return nullcontext()
+
+
+
 # Copied from transformers.models.llama.modeling_llama._get_unpad_data
 def _get_unpad_data(attention_mask):
     seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
@@ -96,41 +107,68 @@ class Qwen2RMSNorm(nn.Module):
         return self.weight * hidden_states.to(input_dtype)
 
 
-# Copied from transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding with Mistral->Qwen2
 class Qwen2RotaryEmbedding(nn.Module):
-    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
+    inv_freq: torch.Tensor  # fix linting for `register_buffer`
+
+    def __init__(self, config: Qwen2Config, device=None):
         super().__init__()
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
 
-        self.dim = dim
-        self.max_position_embeddings = max_position_embeddings
-        self.base = base
-        inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float().to(device) / self.dim))
-        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self.config = config
 
-        # Build here to make `torch.jit.trace` work.
-        self._set_cos_sin_cache(
-            seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype()
-        )
+        self.rope_type = "default"
+        rope_init_fn: Callable = self.compute_default_rope_parameters
+        if self.rope_type != "default":
+            rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
+        inv_freq, self.attention_scaling = rope_init_fn(self.config, device)
 
-    def _set_cos_sin_cache(self, seq_len, device, dtype):
-        self.max_seq_len_cached = seq_len
-        t = torch.arange(self.max_seq_len_cached, device=device, dtype=torch.int64).type_as(self.inv_freq)
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False)
 
-        freqs = torch.outer(t, self.inv_freq)
-        # Different from paper, but it uses a different permutation in order to obtain the same calculation
-        emb = torch.cat((freqs, freqs), dim=-1)
-        self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
-        self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
+    @staticmethod
+    def compute_default_rope_parameters(
+        config: Qwen2Config | None = None,
+        device: Optional["torch.device"] = None,
+        seq_len: int | None = None,
+    ) -> tuple["torch.Tensor", float]:
+        """
+        Computes the inverse frequencies according to the original RoPE implementation
+        Args:
+            config ([`~transformers.PreTrainedConfig`]):
+                The model configuration.
+            device (`torch.device`):
+                The device to use for initialization of the inverse frequencies.
+            seq_len (`int`, *optional*):
+                The current sequence length. Unused for this type of RoPE.
+        Returns:
+            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
+            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
+        """
+        base = config.rope_theta
+        dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
 
-    def forward(self, x, seq_len=None):
-        # x: [bs, num_attention_heads, seq_len, head_size]
-        if seq_len > self.max_seq_len_cached:
-            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype)
+        attention_factor = 1.0  # Unused in this type of RoPE
 
-        return (
-            self.cos_cached[:seq_len].to(dtype=x.dtype),
-            self.sin_cached[:seq_len].to(dtype=x.dtype),
+        # Compute the inverse frequencies
+        inv_freq = 1.0 / (
+            base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim)
         )
+        return inv_freq, attention_factor
+
+    @torch.no_grad()
+    def forward(self, x, position_ids):
+        inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)
+        position_ids_expanded = position_ids[:, None, :].float()
+
+        device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
+        with maybe_autocast(device_type=device_type, enabled=False):  # Force float32
+            freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
+            emb = torch.cat((freqs, freqs), dim=-1)
+            cos = emb.cos() * self.attention_scaling
+            sin = emb.sin() * self.attention_scaling
+
+        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
 
 
 # Copied from transformers.models.llama.modeling_llama.rotate_half
@@ -163,8 +201,8 @@ def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1):
     Returns:
         `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
     """
-    cos = cos[position_ids].unsqueeze(unsqueeze_dim)
-    sin = sin[position_ids].unsqueeze(unsqueeze_dim)
+    cos = cos.unsqueeze(unsqueeze_dim)
+    sin = sin.unsqueeze(unsqueeze_dim)
     q_embed = (q * cos) + (rotate_half(q) * sin)
     k_embed = (k * cos) + (rotate_half(k) * sin)
     return q_embed, k_embed
@@ -235,12 +273,7 @@ class Qwen2Attention(nn.Module):
         self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True)
         self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True)
         self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
-
-        self.rotary_emb = Qwen2RotaryEmbedding(
-            self.head_dim,
-            max_position_embeddings=self.max_position_embeddings,
-            base=self.rope_theta,
-        )
+        self.rotary_emb = Qwen2RotaryEmbedding(self.config)
 
     def forward(
         self,
@@ -277,7 +310,7 @@ class Qwen2Attention(nn.Module):
             # kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
             past_len = past_key_value.get_seq_length(self.layer_idx) if past_key_value is not None else 0
             kv_seq_len += past_len
-        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        cos, sin = self.rotary_emb(value_states, position_ids)
         query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
 
         if past_key_value is not None:
@@ -385,8 +418,7 @@ class Qwen2FlashAttention2(Qwen2Attention):
             kv_seq_len += past_len
 
         # Because the input can be padded, the absolute sequence length depends on the max position id.
-        rotary_seq_len = max(kv_seq_len, position_ids[:, -1].max().item()) + 1
-        cos, sin = self.rotary_emb(value_states, seq_len=rotary_seq_len)
+        cos, sin = self.rotary_emb(value_states, position_ids)
 
         query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
 
@@ -683,7 +715,7 @@ class Qwen2SdpaAttention(Qwen2Attention):
             # kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
             past_len = past_key_value.get_seq_length(self.layer_idx) if past_key_value is not None else 0
             kv_seq_len += past_len
-        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        cos, sin = self.rotary_emb(value_states, position_ids)
 
         query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
 
@@ -842,17 +874,6 @@ class Qwen2PreTrainedModel(PreTrainedModel):
     _supports_sdpa = True
     _supports_cache_class = True
 
-    def _init_weights(self, module):
-        std = self.config.initializer_range
-        if isinstance(module, nn.Linear):
-            module.weight.data.normal_(mean=0.0, std=std)
-            if module.bias is not None:
-                module.bias.data.zero_()
-        elif isinstance(module, nn.Embedding):
-            module.weight.data.normal_(mean=0.0, std=std)
-            if module.padding_idx is not None:
-                module.weight.data[module.padding_idx].zero_()
-
 
 QWEN2_INPUTS_DOCSTRING = r"""
     Args:

tokenization_qwen.py

@@ -1,8 +1,23 @@
-
+# Copyright 2024 The Qwen team, Alibaba Group and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes for Qwen2."""
 from typing import List, Optional
-from transformers.models.qwen2.tokenization_qwen2 import Qwen2Tokenizer as OriginalQwen2Tokenizer
-from transformers.models.qwen2.tokenization_qwen2_fast import Qwen2TokenizerFast as OriginalQwen2TokenizerFast
-from tokenizers import processors
+
+from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers
+from tokenizers.models import BPE
+from tokenizers.processors import TemplateProcessing
+
 
 VOCAB_FILES_NAMES = {
     "vocab_file": "vocab.json",
@@ -10,258 +25,265 @@ VOCAB_FILES_NAMES = {
     "tokenizer_file": "tokenizer.json",
 }
 
-class Qwen2Tokenizer(OriginalQwen2Tokenizer):
-    """
-    Construct a Qwen2 tokenizer. Based on byte-level Byte-Pair-Encoding.
-
-    Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
-    be encoded differently whether it is at the beginning of the sentence (without space) or not:
-
-    ```python
-    >>> from transformers import Qwen2Tokenizer
-
-    >>> tokenizer = Qwen2Tokenizer.from_pretrained("Qwen/Qwen-tokenizer")
-    >>> tokenizer("Hello world")["input_ids"]
-    [9707, 1879]
-
-    >>> tokenizer(" Hello world")["input_ids"]
-    [21927, 1879]
-    ```
-    This is expected.
-
-    You should not use GPT2Tokenizer instead, because of the different pretokenization rules.
-
-    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
-    this superclass for more information regarding those methods.
-
-    Args:
-        vocab_file (`str`):
-            Path to the vocabulary file.
-        merges_file (`str`):
-            Path to the merges file.
-        errors (`str`, *optional*, defaults to `"replace"`):
-            Paradigm to follow when decoding bytes to UTF-8. See
-            [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
-        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
-            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
-            token instead.
-        bos_token (`str`, *optional*):
-            The beginning of sequence token. Not applicable for this tokenizer.
-        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
-            The end of sequence token.
-        pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
-            The token used for padding, for example when batching sequences of different lengths.
-        clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
-            Whether or not the model should cleanup the spaces that were added when splitting the input text during the
-            tokenization process. Not applicable to this tokenizer, since tokenization does not add spaces.
-        split_special_tokens (`bool`, *optional*, defaults to `False`):
-            Whether or not the special tokens should be split during the tokenization process. The default behavior is
-            to not split special tokens. This means that if `<|endoftext|>` is the `eos_token`, then `tokenizer.tokenize("<|endoftext|>") =
-            ['<|endoftext|>`]. Otherwise, if `split_special_tokens=True`, then `tokenizer.tokenize("<|endoftext|>")` will be give `['<',
-            '|', 'endo', 'ft', 'ext', '|', '>']`. This argument is only supported for `slow` tokenizers for the moment.
-        add_eos_token (`bool`, *optional*, defaults to `False`):
-            Whether or not to add an `eos_token` at the end of sequences.
-    """
-
-    def __init__(
-        self,
-        vocab_file,
-        merges_file,
-        errors="replace",
-        unk_token="<|endoftext|>",
-        bos_token=None,
-        eos_token="<|endoftext|>",
-        pad_token="<|endoftext|>",
-        clean_up_tokenization_spaces=False,
-        split_special_tokens=False,
-        add_eos_token=False,
-        **kwargs,
-    ):
-        # The add_eos_token code was inspired by the LlamaTokenizer
-        self.add_eos_token = add_eos_token
-
-        super().__init__(
-            vocab_file=vocab_file,
-            merges_file=merges_file,
-            errors=errors,
-            unk_token=unk_token,
-            bos_token=bos_token,
-            eos_token=eos_token,
-            pad_token=pad_token,
-            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
-            split_special_tokens=split_special_tokens,
-            add_eos_token=add_eos_token,
+MAX_MODEL_INPUT_SIZES = {"qwen/qwen-tokenizer": 32768}
+
+PRETOKENIZE_REGEX = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
+
+from packaging.version import Version
+import transformers
+
+if Version(transformers.__version__) >= Version("5.0.0"):
+    from transformers import TokenizersBackend
+
+    class Qwen2Tokenizer(TokenizersBackend):
+        vocab_files_names = VOCAB_FILES_NAMES
+        model_input_names = ["input_ids", "attention_mask"]
+        model = BPE
+
+        def __init__(
+            self,
+            vocab: str | dict[str, int] | None = None,
+            merges: str | list[str] | None = None,
+            unk_token: str = "<|endoftext|>",
+            bos_token=None,
+            eos_token: str = "<|endoftext|>",
+            pad_token: str = "<|endoftext|>",
+            add_prefix_space=None,
+            add_eos_token=True,
             **kwargs,
-        )
+        ):
+            self.add_prefix_space = add_prefix_space if add_prefix_space is not None else False
+            self._vocab = (
+                vocab
+                if vocab is not None
+                else {
+                    "<|endoftext|>": 0,
+                }
+            )
+            self._merges = merges or []
+            self._tokenizer = Tokenizer(
+                BPE(
+                    vocab=self._vocab,
+                    merges=self._merges,
+                    dropout=None,
+                    unk_token=None,
+                    continuing_subword_prefix="",
+                    end_of_word_suffix="",
+                    fuse_unk=False,
+                    byte_fallback=False,
+                )
+            )
+            self._tokenizer.decoder = decoders.ByteLevel()
+            self._tokenizer.normalizer = normalizers.NFC()
+            self._tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
+                [
+                    pre_tokenizers.Split(
+                        Regex(PRETOKENIZE_REGEX),
+                        behavior="isolated",
+                        invert=False,
+                    ),
+                    pre_tokenizers.ByteLevel(
+                        add_prefix_space=self.add_prefix_space,
+                        use_regex=False,
+                    ),
+                ]
+            )
+
+            super().__init__(
+                unk_token=unk_token,
+                bos_token=bos_token,
+                eos_token=eos_token,
+                pad_token=pad_token,
+                add_prefix_space=add_prefix_space,
+                **kwargs,
+            )
 
-    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
-        eos_token_id = [self.eos_token_id] if self.add_eos_token else []
+            self.add_tokens([AddedToken(token, special=True) for token in self.all_special_tokens])
+            self._add_eos_token = add_eos_token
+            self.update_post_processor()
 
-        output = token_ids_0 + eos_token_id
+        @property
+        def add_eos_token(self):
+            return self._add_eos_token
 
-        if token_ids_1 is not None:
-            output = output + token_ids_1 + eos_token_id
+        def update_post_processor(self):
+            eos = self.eos_token
+            eos_token_id = self.eos_token_id
+            if eos is None and self.add_eos_token:
+                raise ValueError("add_eos_token = True but eos_token = None")
 
-        return output
+            single = f"$A:0{(' ' + eos + ':0') if self.add_eos_token else ''}"
+            pair = f"{single} $B:1{(' ' + eos + ':1') if self.add_eos_token else ''}"
 
-    def get_special_tokens_mask(
-        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
-    ) -> List[int]:
+            special_tokens = []
+            if self.add_eos_token:
+                special_tokens.append((eos, eos_token_id))
+            self._tokenizer.post_processor = TemplateProcessing(
+                single=single, pair=pair, special_tokens=special_tokens
+            )
+
+else:
+    from transformers.models.qwen2.tokenization_qwen2 import Qwen2Tokenizer as OriginalQwen2Tokenizer
+
+    class Qwen2Tokenizer(OriginalQwen2Tokenizer):
         """
-        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
-        special tokens using the tokenizer `prepare_for_model` method.
+        Construct a Qwen2 tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+        Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
+        be encoded differently whether it is at the beginning of the sentence (without space) or not:
+
+        ```python
+        >>> from transformers import Qwen2Tokenizer
+
+        >>> tokenizer = Qwen2Tokenizer.from_pretrained("Qwen/Qwen-tokenizer")
+        >>> tokenizer("Hello world")["input_ids"]
+        [9707, 1879]
+
+        >>> tokenizer(" Hello world")["input_ids"]
+        [21927, 1879]
+        ```
+        This is expected.
+
+        You should not use GPT2Tokenizer instead, because of the different pretokenization rules.
+
+        This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+        this superclass for more information regarding those methods.
 
         Args:
-            token_ids_0 (`List[int]`):
-                List of IDs.
-            token_ids_1 (`List[int]`, *optional*):
-                Optional second list of IDs for sequence pairs.
-            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
-                Whether or not the token list is already formatted with special tokens for the model.
-
-        Returns:
-            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+            vocab_file (`str`):
+                Path to the vocabulary file.
+            merges_file (`str`):
+                Path to the merges file.
+            errors (`str`, *optional*, defaults to `"replace"`):
+                Paradigm to follow when decoding bytes to UTF-8. See
+                [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+            unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+                The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+                token instead.
+            bos_token (`str`, *optional*):
+                The beginning of sequence token. Not applicable for this tokenizer.
+            eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+                The end of sequence token.
+            pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+                The token used for padding, for example when batching sequences of different lengths.
+            clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
+                Whether or not the model should cleanup the spaces that were added when splitting the input text during the
+                tokenization process. Not applicable to this tokenizer, since tokenization does not add spaces.
+            split_special_tokens (`bool`, *optional*, defaults to `False`):
+                Whether or not the special tokens should be split during the tokenization process. The default behavior is
+                to not split special tokens. This means that if `<|endoftext|>` is the `eos_token`, then `tokenizer.tokenize("<|endoftext|>") =
+                ['<|endoftext|>`]. Otherwise, if `split_special_tokens=True`, then `tokenizer.tokenize("<|endoftext|>")` will be give `['<',
+                '|', 'endo', 'ft', 'ext', '|', '>']`. This argument is only supported for `slow` tokenizers for the moment.
+            add_eos_token (`bool`, *optional*, defaults to `False`):
+                Whether or not to add an `eos_token` at the end of sequences.
         """
-        if already_has_special_tokens:
-            return super().get_special_tokens_mask(
-                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
+
+        def __init__(
+                self,
+                vocab_file,
+                merges_file,
+                errors="replace",
+                unk_token="<|endoftext|>",
+                bos_token=None,
+                eos_token="<|endoftext|>",
+                pad_token="<|endoftext|>",
+                clean_up_tokenization_spaces=False,
+                split_special_tokens=False,
+                add_eos_token=False,
+                **kwargs,
+        ):
+            # The add_eos_token code was inspired by the LlamaTokenizer
+            self.add_eos_token = add_eos_token
+
+            super().__init__(
+                vocab_file=vocab_file,
+                merges_file=merges_file,
+                errors=errors,
+                unk_token=unk_token,
+                bos_token=bos_token,
+                eos_token=eos_token,
+                pad_token=pad_token,
+                clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+                split_special_tokens=split_special_tokens,
+                add_eos_token=add_eos_token,
+                **kwargs,
             )
 
-        eos_token_id = [1] if self.add_eos_token else []
+        def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+            eos_token_id = [self.eos_token_id] if self.add_eos_token else []
+
+            output = token_ids_0 + eos_token_id
+
+            if token_ids_1 is not None:
+                output = output + token_ids_1 + eos_token_id
+
+            return output
+
+        def get_special_tokens_mask(
+                self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None,
+                already_has_special_tokens: bool = False
+        ) -> List[int]:
+            """
+            Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
+            special tokens using the tokenizer `prepare_for_model` method.
+
+            Args:
+                token_ids_0 (`List[int]`):
+                    List of IDs.
+                token_ids_1 (`List[int]`, *optional*):
+                    Optional second list of IDs for sequence pairs.
+                already_has_special_tokens (`bool`, *optional*, defaults to `False`):
+                    Whether or not the token list is already formatted with special tokens for the model.
+
+            Returns:
+                `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+            """
+            if already_has_special_tokens:
+                return super().get_special_tokens_mask(
+                    token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
+                )
+
+            eos_token_id = [1] if self.add_eos_token else []
+
+            if token_ids_1 is None:
+                return ([0] * len(token_ids_0)) + eos_token_id
+            return (
+                    ([0] * len(token_ids_0))
+                    + eos_token_id
+                    + ([0] * len(token_ids_1))
+                    + eos_token_id
+            )
 
-        if token_ids_1 is None:
-            return ([0] * len(token_ids_0)) + eos_token_id
-        return (
-            ([0] * len(token_ids_0))
-            + eos_token_id
-            + ([0] * len(token_ids_1))
-            + eos_token_id
-        )
+        def create_token_type_ids_from_sequences(
+                self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+        ) -> List[int]:
+            """
+            Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
+            sequence pair mask has the following format:
 
-    def create_token_type_ids_from_sequences(
-        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
-    ) -> List[int]:
-        """
-        Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
-        sequence pair mask has the following format:
+            ```
+            0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
+            | first sequence    | second sequence |
+            ```
 
-        ```
-        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
-        | first sequence    | second sequence |
-        ```
+            if token_ids_1 is None, only returns the first portion of the mask (0s).
 
-        if token_ids_1 is None, only returns the first portion of the mask (0s).
+            Args:
+                token_ids_0 (`List[int]`):
+                    List of ids.
+                token_ids_1 (`List[int]`, *optional*):
+                    Optional second list of IDs for sequence pairs.
 
-        Args:
-            token_ids_0 (`List[int]`):
-                List of ids.
-            token_ids_1 (`List[int]`, *optional*):
-                Optional second list of IDs for sequence pairs.
+            Returns:
+                `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
+            """
+            eos_token_id = [self.eos_token_id] if self.add_eos_token else []
 
-        Returns:
-            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
-        """
-        eos_token_id = [self.eos_token_id] if self.add_eos_token else []
-
-        output = [0] * len(token_ids_0 + eos_token_id)
-
-        if token_ids_1 is not None:
-            output += [1] * len(token_ids_1 + eos_token_id)
-
-        return output
-
-class Qwen2TokenizerFast(OriginalQwen2TokenizerFast):
-    """
-    Construct a "fast" Qwen2 tokenizer (backed by HuggingFace's *tokenizers* library). Based on byte-level
-    Byte-Pair-Encoding.
-
-    Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
-    be encoded differently whether it is at the beginning of the sentence (without space) or not:
-
-    ```python
-    >>> from transformers import Qwen2TokenizerFast
-
-    >>> tokenizer = Qwen2TokenizerFast.from_pretrained("Qwen/Qwen-tokenizer")
-    >>> tokenizer("Hello world")["input_ids"]
-    [9707, 1879]
-
-    >>> tokenizer(" Hello world")["input_ids"]
-    [21927, 1879]
-    ```
-    This is expected.
-
-    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
-    refer to this superclass for more information regarding those methods.
-
-    Args:
-        vocab_file (`str`, *optional*):
-            Path to the vocabulary file.
-        merges_file (`str`, *optional*):
-            Path to the merges file.
-        tokenizer_file (`str`, *optional*):
-            Path to [tokenizers](https://github.com/huggingface/tokenizers) file (generally has a .json extension) that
-            contains everything needed to load the tokenizer.
-        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
-            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
-            token instead. Not applicable to this tokenizer.
-        bos_token (`str`, *optional*):
-            The beginning of sequence token. Not applicable for this tokenizer.
-        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
-            The end of sequence token.
-        pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
-            The token used for padding, for example when batching sequences of different lengths.
-        add_eos_token (`bool`, *optional*, defaults to `False`):
-            Whether or not to add an `eos_token` at the end of sequences.
-    """
-
-    slow_tokenizer_class = Qwen2Tokenizer
-    padding_side = "left"
-
-    def __init__(
-        self,
-        vocab_file=None,
-        merges_file=None,
-        tokenizer_file=None,
-        unk_token="<|endoftext|>",
-        bos_token=None,
-        eos_token="<|endoftext|>",
-        pad_token="<|endoftext|>",
-        add_eos_token=False,
-        **kwargs,
-    ):
-        super().__init__(
-            vocab_file=vocab_file,
-            merges_file=merges_file,
-            tokenizer_file=tokenizer_file,
-            unk_token=unk_token,
-            bos_token=bos_token,
-            eos_token=eos_token,
-            pad_token=pad_token,
-            **kwargs,
-        )
+            output = [0] * len(token_ids_0 + eos_token_id)
 
-        self._add_eos_token = add_eos_token
-        self.update_post_processor()
+            if token_ids_1 is not None:
+                output += [1] * len(token_ids_1 + eos_token_id)
 
-    def update_post_processor(self):
-        """
-        Updates the underlying post processor with the current `eos_token`.
-        """
-        eos = self.eos_token
-        eos_token_id = self.eos_token_id
-        if eos is None and self.add_eos_token:
-            raise ValueError("add_eos_token = True but eos_token = None")
-
-        single = f"$A:0{(' '+eos+':0') if self.add_eos_token else ''}"
-        pair = f"{single} $B:1{(' '+eos+':1') if self.add_eos_token else ''}"
-
-        special_tokens = []
-        if self.add_eos_token:
-            special_tokens.append((eos, eos_token_id))
-        self._tokenizer.post_processor = processors.TemplateProcessing(
-            single=single, pair=pair, special_tokens=special_tokens
-        )
-
-    @property
-    def add_eos_token(self):
-        return self._add_eos_token
+            return output
+
+__all__ = ["Qwen2Tokenizer"]

tokenizer_config.json

@@ -32,7 +32,7 @@
     "<|im_end|>"
   ],
   "auto_map": {
-    "AutoTokenizer": ["tokenization_qwen.Qwen2Tokenizer", "tokenization_qwen.Qwen2TokenizerFast"]
+    "AutoTokenizer": ["tokenization_qwen.Qwen2Tokenizer", "tokenization_qwen.Qwen2Tokenizer"]
   },
   "bos_token": null,
   "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
@@ -43,5 +43,6 @@
   "pad_token": "<|endoftext|>",
   "split_special_tokens": false,
   "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
+  "unk_token": null,
+  "padding_side": "left"
 }