src/universal_refactor/tokenizer.py

from transformers import AutoTokenizer

def get_tokenizer(name='Salesforce/codegen-350M-multi'):
    try:
        return AutoTokenizer.from_pretrained(name)
    except Exception:
        return AutoTokenizer.from_pretrained('gpt2')