LuisMBA commited on
Commit
14ac4f1
·
verified ·
1 Parent(s): b5056e8

Update fine_tune.py

Browse files
Files changed (1) hide show
  1. fine_tune.py +1 -1
fine_tune.py CHANGED
@@ -16,7 +16,7 @@ model = AutoModelForCausalLM.from_pretrained(model_name)
16
  def preprocess_function(examples):
17
  inputs = [f"Q: {q} A:" for q in examples["question"]]
18
  outputs = examples["answer"]
19
- model_inputs = tokenizer(inputs, text_target=outputs, max_length=512, padding ='max_length', truncation=True)
20
  return model_inputs
21
 
22
  tokenized_dataset = dataset["train"].map(preprocess_function, batched=True)
 
16
  def preprocess_function(examples):
17
  inputs = [f"Q: {q} A:" for q in examples["question"]]
18
  outputs = examples["answer"]
19
+ model_inputs = tokenizer(inputs, text_target=outputs, max_length=512, padding ='longest', truncation=True)
20
  return model_inputs
21
 
22
  tokenized_dataset = dataset["train"].map(preprocess_function, batched=True)