flopml
/

mingru

Model card Files Files and versions

flpelerin commited on Nov 1, 2024

Commit

87a4d20

·

1 Parent(s): 29a6938

fix

Files changed (1) hide show

train.py +1 -1

train.py CHANGED Viewed

@@ -44,7 +44,7 @@ def process_function(examples):
         examples['text'],
         padding='max_length',  # Fixed padding
         truncation=True,
-        max_length=seq_length  # Fixed max length
     )
 tokenized_datasets = dataset.map(process_function, batched=True)

         examples['text'],
         padding='max_length',  # Fixed padding
         truncation=True,
+        max_length=(seq_length * batch_size)  # Fixed max length
     )
 tokenized_datasets = dataset.map(process_function, batched=True)