File size: 232 Bytes
b19c92c |
1 2 3 4 5 6 7 8 |
from transformers import AutoTokenizer
def get_tokenizer(name='Salesforce/codegen-350M-multi'):
try:
return AutoTokenizer.from_pretrained(name)
except Exception:
return AutoTokenizer.from_pretrained('gpt2')
|