Create preprocess Dataset

preprocess Dataset

@@ -0,0 +1,15 @@
+from datasets import load_dataset
+
+# Load your custom dataset (ensure it's in the proper format)
+dataset = load_dataset('text', data_files={'train': 'train.txt', 'test': 'test.txt'})
+
+# Load the GPT-2 tokenizer
+from transformers import GPT2Tokenizer
+
+tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
+
+# Preprocess the dataset
+def preprocess_function(examples):
+    return tokenizer(examples['text'], padding='max_length', truncation=True)
+
+encoded_dataset = dataset.map(preprocess_function, batched=True)