Saving weights and logs of step 1000

config.json

@@ -19,7 +19,7 @@
   "num_hidden_layers": 12,
   "pad_token_id": 1,
   "position_embedding_type": "absolute",
-  "transformers_version": "4.15.0.dev0",
+  "transformers_version": "4.16.0.dev0",
   "type_vocab_size": 1,
   "use_cache": true,
   "vocab_size": 50265

events.out.tfevents.1644205555.t1v-n-79f0077b-w-0.401125.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2af1fe635d7f82065c5bf878b782423969184c0166560393084266ea5901947
+size 40

events.out.tfevents.1644212923.t1v-n-79f0077b-w-0.446891.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8111d41e059aec555485bed7138f0db88eecc16f5892eb6e19b5165841ec58db
+size 40

events.out.tfevents.1644215479.t1v-n-79f0077b-w-0.450647.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:181bc17df02b41dbf683a685073aeb91c09fed6dec3b8ce74e09577c46de1971
+size 147136

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4abb41156cf5e1bcf659c487aa968be2612b5af34c29a3e312dedf77fe42746c
+oid sha256:1c3043126169385088e6f98e40ad48982f05b396f4317b4be26034416ef2fcdc
 size 498796983

run_128_scandinavian.sh

@@ -1,3 +1,7 @@
+#    --per_device_train_batch_size="232" \
+#    --per_device_eval_batch_size="232" \
+
+
 python run_mlm_flax.py \
     --output_dir="./" \
     --model_type="roberta" \
@@ -8,8 +12,8 @@ python run_mlm_flax.py \
     --cache_dir="/mnt/disks/flaxdisk/cache/" \
     --max_seq_length="128" \
     --weight_decay="0.01" \
-    --per_device_train_batch_size="232" \
-    --per_device_eval_batch_size="232" \
+    --per_device_train_batch_size="116" \
+    --per_device_eval_batch_size="116" \
     --pad_to_max_length \
     --learning_rate="3e-4" \
     --warmup_steps="10000" \