Saving weights and logs of step 10000

events.out.tfevents.1641311691.t1v-n-ccbf3e94-w-0.469726.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2498bac991bbea8880117e88bfedd180928eaf9b4314c79bbfcb7e0c0044f371
+size 1470136

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:615e0a78a29e613d4f6f49773083366860c3f4fbf5dda3cd73a84169550ecd46
+oid sha256:a3baa61c5e3349071ae6fc708810d2f500f7aa33a293f7ea847507c8210b383f
 size 498796983

run_mlm_flax.py

@@ -129,6 +129,10 @@ class DataTrainingArguments:
     static_learning_rate: bool = field(
         default=False, metadata={"help": "Use a non decaying learning rate"}
     )
+    end_learning_rate: float = field(
+        default=0, metadata={"help": "End learning rate. Will be ignored it startic learning rate is set"}
+    )
+    
     auth_token: bool = field(
         default=False, metadata={"help": "Use authorisation token"}
     )
@@ -510,6 +514,8 @@ if __name__ == "__main__":
 
     if data_args.static_learning_rate:
         end_lr_value = training_args.learning_rate
+    elif data_args.end_learning_rate:
+        end_lr_value = data_args.end_learning_rate
     else:
         end_lr_value = 0
 

run_step2.sh

@@ -12,6 +12,7 @@
     --per_device_train_batch_size="40" \
     --per_device_eval_batch_size="40" \
     --learning_rate="2e-4" \
+    --end_learning_rate="1e-4" \
     --warmup_steps="0" \
     --overwrite_output_dir \
     --num_train_epochs="2" \
@@ -23,6 +24,5 @@
     --eval_steps="10000" \
     --preprocessing_num_workers="64" \
     --auth_token="True" \
-    --static_learning_rate="True" \
     --dtype="bfloat16" \
     --push_to_hub