Update tokenization_internlm3.py
Browse files
tokenization_internlm3.py
CHANGED
|
@@ -67,7 +67,7 @@ class InternLM3Tokenizer(PreTrainedTokenizer):
|
|
| 67 |
Whether or not to add an initial space to the input. This allows to treat the leading word just as any
|
| 68 |
other word. Again, this should be set with `from_slow=True` to make sure it's taken into account.
|
| 69 |
"""
|
| 70 |
-
|
| 71 |
vocab_files_names = VOCAB_FILES_NAMES
|
| 72 |
model_input_names = ["input_ids", "attention_mask"]
|
| 73 |
|
|
|
|
| 67 |
Whether or not to add an initial space to the input. This allows to treat the leading word just as any
|
| 68 |
other word. Again, this should be set with `from_slow=True` to make sure it's taken into account.
|
| 69 |
"""
|
| 70 |
+
_auto_class = "AutoTokenizer"
|
| 71 |
vocab_files_names = VOCAB_FILES_NAMES
|
| 72 |
model_input_names = ["input_ids", "attention_mask"]
|
| 73 |
|