transformers / utils /test_module /custom_tokenization_fast.py
AbdulElahGwaith's picture
Upload folder using huggingface_hub
a9bd396 verified
from transformers import BertTokenizerFast
from .custom_tokenization import CustomTokenizer
class CustomTokenizerFast(BertTokenizerFast):
slow_tokenizer_class = CustomTokenizer
_auto_map = {
"AutoTokenizer": ("custom_tokenization.CustomTokenizer", "custom_tokenization_fast.CustomTokenizerFast")
}