saracandu commited on
Commit
3b2a37f
·
verified ·
1 Parent(s): 92a2d05
Files changed (1) hide show
  1. modeling.py +3 -1
modeling.py CHANGED
@@ -773,7 +773,7 @@ class STLTokenizer(PreTrainedTokenizer):
773
  """
774
 
775
  def __init__(self, vocab_path: str, unk_token: str = "unk", pad_token: str = "pad",
776
- bos_token: str = "/s", eos_token: str = "s", model_max_length = 512):
777
  """
778
  Initializes the STLTokenizer with a given vocabulary and special tokens.
779
 
@@ -791,6 +791,8 @@ class STLTokenizer(PreTrainedTokenizer):
791
  self.eos_token = eos_token
792
  self.model_max_length = model_max_length
793
  self.id_to_token = {v: k for k, v in self.vocab.items()} # Reverse mapping
 
 
794
 
795
  @property
796
  def vocab_size(self) -> int:
 
773
  """
774
 
775
  def __init__(self, vocab_path: str, unk_token: str = "unk", pad_token: str = "pad",
776
+ bos_token: str = "/s", eos_token: str = "s", model_max_length = 512, *args, **kwargs):
777
  """
778
  Initializes the STLTokenizer with a given vocabulary and special tokens.
779
 
 
791
  self.eos_token = eos_token
792
  self.model_max_length = model_max_length
793
  self.id_to_token = {v: k for k, v in self.vocab.items()} # Reverse mapping
794
+ super().__init__(unk_token=unk_token, pad_token=pad_token, bos_token=bos_token, eos_token=eos_token,
795
+ model_max_length=model_max_length, *args, **kwargs)
796
 
797
  @property
798
  def vocab_size(self) -> int: