File size: 564 Bytes
f9cc63a |
1 2 3 4 5 6 7 8 9 10 11 12 |
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
model_ckpt = "codeparrot"
org = "transformersbook"
def model_size(model):
return sum(t.numel() for t in model.parameters())
tokenizer = AutoTokenizer.from_pretrained(org + "/" + model_ckpt)
config_small = AutoConfig.from_pretrained("gpt2", vocab_size=len(tokenizer))
model_small = AutoModelForCausalLM.from_config(config_small)
print(f'GPT-2 size: {model_size(model_small)/1000**2:.1f}M parameters')
model_small.save_pretrained("models/" + model_ckpt + "-small", push_to_hub=True) |