Update app.py

app.py

@@ -17,5 +17,17 @@ def summarize():
 #    return df.head()
     checkpoint = "google-t5/t5-small"
     tokenizer = AutoTokenizer.from_pretrained(checkpoint)
-    
+
+    prefix = "summarize: "
+
+
+    def preprocess_function(examples):
+        inputs = [prefix + doc for doc in examples["text"]]
+        model_inputs = tokenizer(inputs, max_length=1024, truncation=True)
+
+        labels = tokenizer(text_target=examples["summary"], max_length=128, truncation=True)
+
+        model_inputs["labels"] = labels["input_ids"]
+        return model_inputs
+        
     return tokenizer