oweller2 commited on
Commit ·
8e5d4e2
1
Parent(s): 5448b02
fix
Browse files- tokenizer.py +1 -1
tokenizer.py
CHANGED
|
@@ -80,4 +80,4 @@ class ModernDecoderBERTTokenizer(PreTrainedTokenizerFast):
|
|
| 80 |
|
| 81 |
# Register the class
|
| 82 |
from transformers import AutoTokenizer
|
| 83 |
-
AutoTokenizer.register(ModernDecoderBERTTokenizer, fast_tokenizer_class=ModernDecoderBERTTokenizer
|
|
|
|
| 80 |
|
| 81 |
# Register the class
|
| 82 |
from transformers import AutoTokenizer
|
| 83 |
+
AutoTokenizer.register(ModernDecoderBERTTokenizer, fast_tokenizer_class=ModernDecoderBERTTokenizer)
|