Percy3822 commited on
Commit
30ddf38
·
verified ·
1 Parent(s): 55bae34

Update train.py

Browse files
Files changed (1) hide show
  1. train.py +1 -0
train.py CHANGED
@@ -8,6 +8,7 @@ dataset = load_dataset("json", data_files="python.jsonl")
8
  # Load tokenizer and model
9
  model_name = "distilgpt2"
10
  tokenizer = AutoTokenizer.from_pretrained(model_name)
 
11
  model = AutoModelForCausalLM.from_pretrained(model_name)
12
 
13
  # Tokenize function
 
8
  # Load tokenizer and model
9
  model_name = "distilgpt2"
10
  tokenizer = AutoTokenizer.from_pretrained(model_name)
11
+ tokenizer.pad_token = tokenizer.eos_token
12
  model = AutoModelForCausalLM.from_pretrained(model_name)
13
 
14
  # Tokenize function