modeling/tokenizers.py

#
# Zhou Bo

#

""" tokenizers
"""

from .spm_tokenizer import *
from .gpt2_tokenizer import GPT2Tokenizer
from wywLM.models import BertTokenizer

__all__ = ['tokenizers']
tokenizers={
    'gpt2': GPT2Tokenizer,
    'spm': SPMTokenizer,
    'bert': BertTokenizer
    }