Spaces:
Sleeping
Sleeping
Update train.py
Browse files
train.py
CHANGED
|
@@ -8,6 +8,7 @@ dataset = load_dataset("json", data_files="python.jsonl")
|
|
| 8 |
# Load tokenizer and model
|
| 9 |
model_name = "distilgpt2"
|
| 10 |
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
|
|
|
| 11 |
model = AutoModelForCausalLM.from_pretrained(model_name)
|
| 12 |
|
| 13 |
# Tokenize function
|
|
|
|
| 8 |
# Load tokenizer and model
|
| 9 |
model_name = "distilgpt2"
|
| 10 |
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
| 11 |
+
tokenizer.pad_token = tokenizer.eos_token
|
| 12 |
model = AutoModelForCausalLM.from_pretrained(model_name)
|
| 13 |
|
| 14 |
# Tokenize function
|