Spaces:
Paused
Paused
Update tokenization_small100.py
Browse files- tokenization_small100.py +1 -1
tokenization_small100.py
CHANGED
|
@@ -180,7 +180,7 @@ class SMALL100Tokenizer(PreTrainedTokenizer):
|
|
| 180 |
|
| 181 |
@property
|
| 182 |
def vocab_size(self) -> int:
|
| 183 |
-
return len(self.
|
| 184 |
|
| 185 |
@property
|
| 186 |
def tgt_lang(self) -> str:
|
|
|
|
| 180 |
|
| 181 |
@property
|
| 182 |
def vocab_size(self) -> int:
|
| 183 |
+
return len(self.encode) + len(self.lang_token_to_id) + self.num_madeup_words
|
| 184 |
|
| 185 |
@property
|
| 186 |
def tgt_lang(self) -> str:
|