File size: 148 Bytes
9e1f23b
 
 
 
1
2
3
4
from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("your-model-name")
tokenizer.save_pretrained("model_directory")