fix

train.py

@@ -40,7 +40,12 @@ print(f"Tokenzier has {vocab_size} unique tokens")
 dataset = load_dataset(dataset_path)
 
 def process_function(examples):
-    return tokenizer(examples['text'], padding='longest', truncation=True)
+    return tokenizer(
+        examples['text'],
+        padding='max_length',  # Fixed padding
+        truncation=True,
+        max_length=seq_length  # Fixed max length
+    )
 
 tokenized_datasets = dataset.map(process_function, batched=True)
 print(f"Dataset has {tokenized_datasets['train'].num_rows} rows of {batch_size} times {seq_length} tokens")