try
Browse files- modeling.py +3 -1
modeling.py
CHANGED
|
@@ -773,7 +773,7 @@ class STLTokenizer(PreTrainedTokenizer):
|
|
| 773 |
"""
|
| 774 |
|
| 775 |
def __init__(self, vocab_path: str, unk_token: str = "unk", pad_token: str = "pad",
|
| 776 |
-
bos_token: str = "/s", eos_token: str = "s", model_max_length = 512):
|
| 777 |
"""
|
| 778 |
Initializes the STLTokenizer with a given vocabulary and special tokens.
|
| 779 |
|
|
@@ -791,6 +791,8 @@ class STLTokenizer(PreTrainedTokenizer):
|
|
| 791 |
self.eos_token = eos_token
|
| 792 |
self.model_max_length = model_max_length
|
| 793 |
self.id_to_token = {v: k for k, v in self.vocab.items()} # Reverse mapping
|
|
|
|
|
|
|
| 794 |
|
| 795 |
@property
|
| 796 |
def vocab_size(self) -> int:
|
|
|
|
| 773 |
"""
|
| 774 |
|
| 775 |
def __init__(self, vocab_path: str, unk_token: str = "unk", pad_token: str = "pad",
|
| 776 |
+
bos_token: str = "/s", eos_token: str = "s", model_max_length = 512, *args, **kwargs):
|
| 777 |
"""
|
| 778 |
Initializes the STLTokenizer with a given vocabulary and special tokens.
|
| 779 |
|
|
|
|
| 791 |
self.eos_token = eos_token
|
| 792 |
self.model_max_length = model_max_length
|
| 793 |
self.id_to_token = {v: k for k, v in self.vocab.items()} # Reverse mapping
|
| 794 |
+
super().__init__(unk_token=unk_token, pad_token=pad_token, bos_token=bos_token, eos_token=eos_token,
|
| 795 |
+
model_max_length=model_max_length, *args, **kwargs)
|
| 796 |
|
| 797 |
@property
|
| 798 |
def vocab_size(self) -> int:
|