Training in progress, step 4000, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8bf065dce895a71030a8256ac77e86f07171b7a40886a69f80a202e40a5e3c4
+oid sha256:e53425bdc67c0605a853db9bf5bcf93793a8c96cfa5f75f67b2650c95c20081e
 size 2252747

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7d35dbfbf266d337501eed62097b3a3a9e66befb826a069c58e0d8414eae6cb
+oid sha256:d7eac66c44b28b5ff036fd57811c8fd5650cb56fd5b287df7ec209d4660ff3c1
 size 14455

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b716fb1dcbc77ddc6a0ae7c4c98d559e36f1986fff006ef6982bd75247833235
+oid sha256:76cf98a8cde877f1e49a9e4b884c4dce3d51bb4035429d464e74acc35ddca024
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.04,
+  "epoch": 0.08,
   "eval_steps": 1000,
-  "global_step": 2000,
+  "global_step": 4000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -166,6 +166,164 @@
       "eval_samples_per_second": 104.217,
       "eval_steps_per_second": 13.027,
       "step": 2000
+    },
+    {
+      "epoch": 0.042,
+      "grad_norm": NaN,
+      "learning_rate": 0.0016792,
+      "loss": NaN,
+      "step": 2100
+    },
+    {
+      "epoch": 0.044,
+      "grad_norm": NaN,
+      "learning_rate": 0.0017592,
+      "loss": NaN,
+      "step": 2200
+    },
+    {
+      "epoch": 0.046,
+      "grad_norm": NaN,
+      "learning_rate": 0.0018392,
+      "loss": NaN,
+      "step": 2300
+    },
+    {
+      "epoch": 0.048,
+      "grad_norm": NaN,
+      "learning_rate": 0.0019192,
+      "loss": NaN,
+      "step": 2400
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": NaN,
+      "learning_rate": 0.0019992,
+      "loss": NaN,
+      "step": 2500
+    },
+    {
+      "epoch": 0.052,
+      "grad_norm": NaN,
+      "learning_rate": 0.001999978563623903,
+      "loss": NaN,
+      "step": 2600
+    },
+    {
+      "epoch": 0.054,
+      "grad_norm": NaN,
+      "learning_rate": 0.0019999133871331223,
+      "loss": NaN,
+      "step": 2700
+    },
+    {
+      "epoch": 0.056,
+      "grad_norm": NaN,
+      "learning_rate": 0.0019998044711915177,
+      "loss": NaN,
+      "step": 2800
+    },
+    {
+      "epoch": 0.058,
+      "grad_norm": NaN,
+      "learning_rate": 0.0019996518205634257,
+      "loss": NaN,
+      "step": 2900
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": NaN,
+      "learning_rate": 0.0019994554419262797,
+      "loss": NaN,
+      "step": 3000
+    },
+    {
+      "epoch": 0.06,
+      "eval_accuracy": 0.0,
+      "eval_loss": NaN,
+      "eval_runtime": 9.7513,
+      "eval_samples_per_second": 102.551,
+      "eval_steps_per_second": 12.819,
+      "step": 3000
+    },
+    {
+      "epoch": 0.062,
+      "grad_norm": NaN,
+      "learning_rate": 0.001999215343870317,
+      "loss": NaN,
+      "step": 3100
+    },
+    {
+      "epoch": 0.064,
+      "grad_norm": NaN,
+      "learning_rate": 0.0019989315368982054,
+      "loss": NaN,
+      "step": 3200
+    },
+    {
+      "epoch": 0.066,
+      "grad_norm": NaN,
+      "learning_rate": 0.0019986040334245797,
+      "loss": NaN,
+      "step": 3300
+    },
+    {
+      "epoch": 0.068,
+      "grad_norm": NaN,
+      "learning_rate": 0.001998232847775504,
+      "loss": NaN,
+      "step": 3400
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": NaN,
+      "learning_rate": 0.0019978179961878404,
+      "loss": NaN,
+      "step": 3500
+    },
+    {
+      "epoch": 0.072,
+      "grad_norm": NaN,
+      "learning_rate": 0.001997359496808541,
+      "loss": NaN,
+      "step": 3600
+    },
+    {
+      "epoch": 0.074,
+      "grad_norm": NaN,
+      "learning_rate": 0.001996857369693855,
+      "loss": NaN,
+      "step": 3700
+    },
+    {
+      "epoch": 0.076,
+      "grad_norm": NaN,
+      "learning_rate": 0.0019963116368084486,
+      "loss": NaN,
+      "step": 3800
+    },
+    {
+      "epoch": 0.078,
+      "grad_norm": NaN,
+      "learning_rate": 0.001995722322024446,
+      "loss": NaN,
+      "step": 3900
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": NaN,
+      "learning_rate": 0.001995089451120385,
+      "loss": NaN,
+      "step": 4000
+    },
+    {
+      "epoch": 0.08,
+      "eval_accuracy": 0.0,
+      "eval_loss": NaN,
+      "eval_runtime": 9.7912,
+      "eval_samples_per_second": 102.132,
+      "eval_steps_per_second": 12.767,
+      "step": 4000
     }
   ],
   "logging_steps": 100,