File size: 125 Bytes
9a1ddeb |
1 2 3 4 5 |
from transformers import AutoTokenizer
def load_tokenizer(model_name):
return AutoTokenizer.from_pretrained(model_name)
|
9a1ddeb |
1 2 3 4 5 |
from transformers import AutoTokenizer
def load_tokenizer(model_name):
return AutoTokenizer.from_pretrained(model_name)
|