Training in progress, step 76, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b7fe9ffa0583b6f061017a8900f8f9b226202cfe4c7807405c17a772751672c
+oid sha256:8e1fc10a333194a5070fe31d0a4fee76d4e1a5bfd5e90f6f4a165a37847ef76a
 size 128622

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b996ddb827584abfc2d096b8c41b9ea6c92000fbed4975bf65283d683a4ea377
+oid sha256:df83ebf7307bc2a73f9c79f95913d279ac09e32e5566491870d0ceb342f907e0
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eff4e51807fc2798ef7e3176a7ae7a505f7b610c949cf8c1d65662116e17ea14
+oid sha256:e8ddabf60630e29c65bf15d4d38a70b53e1096fcbf23378832f8b10226895763
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": NaN,
   "best_model_checkpoint": "miner_id_24/checkpoint-50",
-  "epoch": 0.004103584737742464,
+  "epoch": 0.006237448801368546,
   "eval_steps": 25,
-  "global_step": 50,
+  "global_step": 76,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -381,6 +381,196 @@
       "eval_samples_per_second": 132.092,
       "eval_steps_per_second": 132.092,
       "step": 50
+    },
+    {
+      "epoch": 0.004185656432497313,
+      "grad_norm": NaN,
+      "learning_rate": 9.916118258515936e-05,
+      "loss": 0.0,
+      "step": 51
+    },
+    {
+      "epoch": 0.004267728127252163,
+      "grad_norm": NaN,
+      "learning_rate": 9.421851671977372e-05,
+      "loss": 0.0,
+      "step": 52
+    },
+    {
+      "epoch": 0.004349799822007012,
+      "grad_norm": NaN,
+      "learning_rate": 8.940340387142462e-05,
+      "loss": 0.0,
+      "step": 53
+    },
+    {
+      "epoch": 0.004431871516761861,
+      "grad_norm": NaN,
+      "learning_rate": 8.47245211989213e-05,
+      "loss": 0.0,
+      "step": 54
+    },
+    {
+      "epoch": 0.004513943211516711,
+      "grad_norm": NaN,
+      "learning_rate": 8.019030036508827e-05,
+      "loss": 0.0,
+      "step": 55
+    },
+    {
+      "epoch": 0.00459601490627156,
+      "grad_norm": NaN,
+      "learning_rate": 7.580891234233904e-05,
+      "loss": 0.0,
+      "step": 56
+    },
+    {
+      "epoch": 0.004678086601026409,
+      "grad_norm": NaN,
+      "learning_rate": 7.158825268803127e-05,
+      "loss": 0.0,
+      "step": 57
+    },
+    {
+      "epoch": 0.0047601582957812584,
+      "grad_norm": NaN,
+      "learning_rate": 6.75359273161392e-05,
+      "loss": 0.0,
+      "step": 58
+    },
+    {
+      "epoch": 0.004842229990536108,
+      "grad_norm": NaN,
+      "learning_rate": 6.365923879088219e-05,
+      "loss": 0.0,
+      "step": 59
+    },
+    {
+      "epoch": 0.004924301685290957,
+      "grad_norm": NaN,
+      "learning_rate": 5.996517316701069e-05,
+      "loss": 0.0,
+      "step": 60
+    },
+    {
+      "epoch": 0.005006373380045806,
+      "grad_norm": NaN,
+      "learning_rate": 5.646038740046304e-05,
+      "loss": 0.0,
+      "step": 61
+    },
+    {
+      "epoch": 0.005088445074800656,
+      "grad_norm": NaN,
+      "learning_rate": 5.315119735208132e-05,
+      "loss": 0.0,
+      "step": 62
+    },
+    {
+      "epoch": 0.005170516769555505,
+      "grad_norm": NaN,
+      "learning_rate": 5.004356640600297e-05,
+      "loss": 0.0,
+      "step": 63
+    },
+    {
+      "epoch": 0.005252588464310354,
+      "grad_norm": NaN,
+      "learning_rate": 4.71430947232396e-05,
+      "loss": 0.0,
+      "step": 64
+    },
+    {
+      "epoch": 0.0053346601590652035,
+      "grad_norm": NaN,
+      "learning_rate": 4.4455009149808265e-05,
+      "loss": 0.0,
+      "step": 65
+    },
+    {
+      "epoch": 0.005416731853820053,
+      "grad_norm": NaN,
+      "learning_rate": 4.1984153797601665e-05,
+      "loss": 0.0,
+      "step": 66
+    },
+    {
+      "epoch": 0.005498803548574902,
+      "grad_norm": NaN,
+      "learning_rate": 3.9734981314971234e-05,
+      "loss": 0.0,
+      "step": 67
+    },
+    {
+      "epoch": 0.005580875243329751,
+      "grad_norm": NaN,
+      "learning_rate": 3.771154486275363e-05,
+      "loss": 0.0,
+      "step": 68
+    },
+    {
+      "epoch": 0.005662946938084601,
+      "grad_norm": NaN,
+      "learning_rate": 3.591749081020113e-05,
+      "loss": 0.0,
+      "step": 69
+    },
+    {
+      "epoch": 0.005745018632839449,
+      "grad_norm": NaN,
+      "learning_rate": 3.435605216397765e-05,
+      "loss": 0.0,
+      "step": 70
+    },
+    {
+      "epoch": 0.005827090327594299,
+      "grad_norm": NaN,
+      "learning_rate": 3.303004274206237e-05,
+      "loss": 0.0,
+      "step": 71
+    },
+    {
+      "epoch": 0.0059091620223491485,
+      "grad_norm": NaN,
+      "learning_rate": 3.194185210305936e-05,
+      "loss": 0.0,
+      "step": 72
+    },
+    {
+      "epoch": 0.005991233717103998,
+      "grad_norm": NaN,
+      "learning_rate": 3.1093441240051626e-05,
+      "loss": 0.0,
+      "step": 73
+    },
+    {
+      "epoch": 0.006073305411858847,
+      "grad_norm": NaN,
+      "learning_rate": 3.048633904675892e-05,
+      "loss": 0.0,
+      "step": 74
+    },
+    {
+      "epoch": 0.006155377106613696,
+      "grad_norm": NaN,
+      "learning_rate": 3.012163956236801e-05,
+      "loss": 0.0,
+      "step": 75
+    },
+    {
+      "epoch": 0.006155377106613696,
+      "eval_loss": NaN,
+      "eval_runtime": 0.3845,
+      "eval_samples_per_second": 130.033,
+      "eval_steps_per_second": 130.033,
+      "step": 75
+    },
+    {
+      "epoch": 0.006237448801368546,
+      "grad_norm": NaN,
+      "learning_rate": 2.9999999999999997e-05,
+      "loss": 0.0,
+      "step": 76
     }
   ],
   "logging_steps": 1,
@@ -395,7 +585,7 @@
         "early_stopping_threshold": 0.0
       },
       "attributes": {
-        "early_stopping_patience_counter": 0
+        "early_stopping_patience_counter": 1
       }
     },
     "TrainerControl": {
@@ -404,12 +594,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 20860580659200.0,
+  "total_flos": 31708082601984.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null