add requirements
Browse files- run_mlm_flax_stream.py +0 -1
run_mlm_flax_stream.py
CHANGED
|
@@ -487,7 +487,6 @@ if __name__ == "__main__":
|
|
| 487 |
def tokenize_function(examples):
|
| 488 |
return tokenizer(
|
| 489 |
examples[data_args.text_column_name],
|
| 490 |
-
max_length=512,
|
| 491 |
return_special_tokens_mask=True
|
| 492 |
)
|
| 493 |
|
|
|
|
| 487 |
def tokenize_function(examples):
|
| 488 |
return tokenizer(
|
| 489 |
examples[data_args.text_column_name],
|
|
|
|
| 490 |
return_special_tokens_mask=True
|
| 491 |
)
|
| 492 |
|