utils.py

import os

from transformers import AutoTokenizer, AutoModelForCausalLM

def get_tok_and_model(path_for_model):
    if not os.path.exists(path_for_model):
        raise RuntimeError("no cached model.")
    tok = AutoTokenizer.from_pretrained(path_for_model, padding_side='left')
    tok.pad_token_id = 50256
    # default for open-ended generation
    model = AutoModelForCausalLM.from_pretrained(path_for_model)
    return tok, model