Spaces:
No application file
No application file
Update finetune.py
Browse files- finetune.py +4 -4
finetune.py
CHANGED
|
@@ -89,12 +89,12 @@ def fine_tune(model_name, finetune_type, model, tokenizer, summarize_text, train
|
|
| 89 |
max_output = 60
|
| 90 |
train_dataset = train
|
| 91 |
eval_dataset = val
|
| 92 |
-
if finetune_type == "
|
| 93 |
-
train_dataset = Dataset.from_pandas(train.sample(1200))
|
| 94 |
-
eval_dataset = Dataset.from_pandas(val.sample(150))
|
| 95 |
-
else:
|
| 96 |
train_dataset = Dataset.from_pandas(train.sample(1500))
|
| 97 |
eval_dataset = Dataset.from_pandas(val.sample(200))
|
|
|
|
|
|
|
|
|
|
| 98 |
def preprocess_function(examples):
|
| 99 |
inputs = [f"Summarize the text: {ex}" for ex in examples["source"]]
|
| 100 |
targets = [f"Summary: {ex}" for ex in examples["target"]]
|
|
|
|
| 89 |
max_output = 60
|
| 90 |
train_dataset = train
|
| 91 |
eval_dataset = val
|
| 92 |
+
if finetune_type == "english":
|
|
|
|
|
|
|
|
|
|
| 93 |
train_dataset = Dataset.from_pandas(train.sample(1500))
|
| 94 |
eval_dataset = Dataset.from_pandas(val.sample(200))
|
| 95 |
+
else:
|
| 96 |
+
train_dataset = Dataset.from_pandas(train.sample(1200))
|
| 97 |
+
eval_dataset = Dataset.from_pandas(val.sample(150))
|
| 98 |
def preprocess_function(examples):
|
| 99 |
inputs = [f"Summarize the text: {ex}" for ex in examples["source"]]
|
| 100 |
targets = [f"Summary: {ex}" for ex in examples["target"]]
|