use proper tokenizer.json and change to batch_size=256

events.out.tfevents.1626444652.t1v-n-bf8aeee7-w-0.7169.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28995d9c2e8ef74fe2f9bb8d05d9f6eacef6a2b734557bcf1d66ef3c32756805
+size 40

run.sh

@@ -9,8 +9,8 @@ python3 run_mlm_flax.py \
     --max_seq_length="128" \
     --weight_decay="0.01" \
     --preprocessing_num_workers="64" \
-    --per_device_train_batch_size="128" \
-    --per_device_eval_batch_size="128" \
+    --per_device_train_batch_size="256" \
+    --per_device_eval_batch_size="256" \
     --learning_rate="3e-5" \
     --warmup_steps="5000" \
     --overwrite_output_dir \
@@ -18,7 +18,8 @@ python3 run_mlm_flax.py \
     --num_train_epochs="8" \
     --adam_beta1="0.9" \
     --adam_beta2="0.98" \
-    --logging_steps="500" \
-    --save_steps="5000" \
-    --eval_steps="20000" \
+    --logging_steps="62" \
+    --save_steps="625" \
+    --eval_steps="2500" \
+    --dtype="bfloat16" \
     --push_to_hub