Training in progress, step 50000, checkpoint

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:214a1072eb6e2e4cc13be55f0e76f496dbf8421eeae52bb353439962f9793e2b
+oid sha256:0abfd4b4cd810dbecfbb741346885f586009f5de461a8977bf4d899f17d11548
 size 14645

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 3.3261267254282387,
+  "epoch": 4.157658406785298,
   "eval_steps": 10000,
-  "global_step": 40000,
+  "global_step": 50000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -600,6 +600,154 @@
       "eval_samples_per_second": 139.771,
       "eval_steps_per_second": 4.368,
       "step": 40000
+    },
+    {
+      "epoch": 3.367703309496092,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 40500
+    },
+    {
+      "epoch": 3.409279893563945,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 41000
+    },
+    {
+      "epoch": 3.4508564776317976,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 41500
+    },
+    {
+      "epoch": 3.492433061699651,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 42000
+    },
+    {
+      "epoch": 3.5340096457675036,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 42500
+    },
+    {
+      "epoch": 3.5755862298353565,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 43000
+    },
+    {
+      "epoch": 3.6171628139032097,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 43500
+    },
+    {
+      "epoch": 3.658739397971063,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 44000
+    },
+    {
+      "epoch": 3.7003159820389158,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 44500
+    },
+    {
+      "epoch": 3.7418925661067686,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 45000
+    },
+    {
+      "epoch": 3.7834691501746214,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 45500
+    },
+    {
+      "epoch": 3.8250457342424746,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 46000
+    },
+    {
+      "epoch": 3.866622318310328,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 46500
+    },
+    {
+      "epoch": 3.9081989023781807,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 47000
+    },
+    {
+      "epoch": 3.9497754864460335,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 47500
+    },
+    {
+      "epoch": 3.9913520705138867,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 48000
+    },
+    {
+      "epoch": 4.03292865458174,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 48500
+    },
+    {
+      "epoch": 4.074505238649593,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 49000
+    },
+    {
+      "epoch": 4.116081822717446,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 49500
+    },
+    {
+      "epoch": 4.157658406785298,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 50000
+    },
+    {
+      "epoch": 4.157658406785298,
+      "eval_loss": NaN,
+      "eval_runtime": 2750.6919,
+      "eval_samples_per_second": 139.902,
+      "eval_steps_per_second": 4.372,
+      "step": 50000
     }
   ],
   "logging_steps": 500,
@@ -619,7 +767,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4.483885876471327e+17,
+  "total_flos": 5.605184286005299e+17,
   "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null