ab0f6ec
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# # Author: penhe@microsoft.com # Date: 04/25/2019 # """ tokenizers """ from .spm_tokenizer import * from .gpt2_tokenizer import GPT2Tokenizer __all__ = ['tokenizers'] tokenizers={ 'gpt2': GPT2Tokenizer, 'spm': SPMTokenizer }
