File size: 280 Bytes
d10ad42 | 1 2 3 4 5 6 7 8 9 10 | """MATILDA tokenizer alias: vocabulary, normalization and special tokens are unchanged."""
from transformers.models.qwen2.tokenization_qwen2 import Qwen2Tokenizer
class MatildaJevTokenizer(Qwen2Tokenizer):
pass
MatildaJevTokenizer.register_for_auto_class("AutoTokenizer")
|