Spaces:
Sleeping
Sleeping
Update fine_tune.py
Browse files- fine_tune.py +1 -1
fine_tune.py
CHANGED
|
@@ -16,7 +16,7 @@ model = AutoModelForCausalLM.from_pretrained(model_name)
|
|
| 16 |
def preprocess_function(examples):
|
| 17 |
inputs = [f"Q: {q} A:" for q in examples["question"]]
|
| 18 |
outputs = examples["answer"]
|
| 19 |
-
model_inputs = tokenizer(inputs, text_target=outputs, max_length=512, padding ='
|
| 20 |
return model_inputs
|
| 21 |
|
| 22 |
tokenized_dataset = dataset["train"].map(preprocess_function, batched=True)
|
|
|
|
| 16 |
def preprocess_function(examples):
|
| 17 |
inputs = [f"Q: {q} A:" for q in examples["question"]]
|
| 18 |
outputs = examples["answer"]
|
| 19 |
+
model_inputs = tokenizer(inputs, text_target=outputs, max_length=512, padding ='longest', truncation=True)
|
| 20 |
return model_inputs
|
| 21 |
|
| 22 |
tokenized_dataset = dataset["train"].map(preprocess_function, batched=True)
|