support transformers==4.34.0

tokenization_plamo.py

@@ -5,7 +5,6 @@ from shutil import copyfile
 from typing import Any, Dict, List, Optional, Tuple
 
 import sentencepiece as spm
-import transformers
 from transformers.tokenization_utils import PreTrainedTokenizer
 from transformers.utils import logging
 
@@ -35,6 +34,12 @@ class PlamoTokenizer(PreTrainedTokenizer):  # type: ignore
             kwargs["add_bos_token"] = False
         if "add_eos_token" not in kwargs:
             kwargs["add_eos_token"] = False
+        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
+        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
+        self.sp_model.Load(vocab_file)
+        self.vocab_file = vocab_file
+        self.add_bos_token = kwargs["add_bos_token"]
+        self.add_eos_token = kwargs["add_eos_token"]
 
         super().__init__(
             vocab_file=vocab_file,
@@ -50,15 +55,6 @@ class PlamoTokenizer(PreTrainedTokenizer):  # type: ignore
             **kwargs,
         )
 
-        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
-        self.vocab_file = vocab_file
-        self.add_bos_token = kwargs["add_bos_token"]
-        self.add_eos_token = kwargs["add_eos_token"]
-        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
-        self.sp_model.Load(vocab_file)
-
-        self.add_tokens(self.all_special_tokens_extended, special_tokens=True)
-
     # the functions below are copied from hf transformers LlamaTokenizer's implementation to fix the behaviour of the tokenizer
     # https://github.com/huggingface/transformers/blob/v4.30.2/src/transformers/models/llama/tokenization_llama.py
 
@@ -155,7 +151,3 @@ class PlamoTokenizer(PreTrainedTokenizer):  # type: ignore
                 fi.write(content_spiece_model)
 
         return (out_vocab_file,)
-
-
-class PlamoConfig(transformers.LlamaConfig):  # type: ignore
-    model_type = "plamo"