Checkpoint at step 30000 (pushed after pausing the host)

run/progress.json

@@ -1,35 +1,35 @@
 {
   "state": "training",
-  "global_step": 30000,
+  "global_step": 38400,
   "max_steps": 129534,
-  "epoch": 0.46320123520329387,
-  "elapsed_seconds": 11153.100829963005,
-  "steps_per_second": 2.689834912942279,
-  "optimizer_updates_this_process": 30000,
+  "epoch": 0.5928975810602162,
+  "elapsed_seconds": 14220.950442084999,
+  "steps_per_second": 2.7002414611023706,
+  "optimizer_updates_this_process": 38400,
   "peak_cuda_GiB": 16.14784526824951,
   "world_size": 1,
   "effective_batch_size": 72,
   "recent_metrics": [
     {
-      "loss": 0.019623750686645507,
-      "grad_norm": 0.4188960790634155,
-      "learning_rate": 1.583136473821545e-05,
-      "epoch": 0.4168811116829645,
-      "step": 27000
+      "loss": 0.01756742477416992,
+      "grad_norm": 0.28143829107284546,
+      "learning_rate": 1.4441768184414904e-05,
+      "epoch": 0.5558414822439527,
+      "step": 36000
     },
     {
-      "loss": 0.018160568237304688,
-      "grad_norm": 0.35333889722824097,
-      "learning_rate": 1.56769651211265e-05,
-      "epoch": 0.4323211528564076,
-      "step": 28000
+      "loss": 0.01727944755554199,
+      "grad_norm": 0.07350185513496399,
+      "learning_rate": 1.4287368567325955e-05,
+      "epoch": 0.5712815234173958,
+      "step": 37000
     },
     {
-      "loss": 0.01974146842956543,
-      "grad_norm": 0.02362905442714691,
-      "learning_rate": 1.5522565504037554e-05,
-      "epoch": 0.44776119402985076,
-      "step": 29000
+      "loss": 0.01676115608215332,
+      "grad_norm": 0.022185837849974632,
+      "learning_rate": 1.4132968950237004e-05,
+      "epoch": 0.5867215645908389,
+      "step": 38000
     }
   ]
 }