Checkpoint at step 110000

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56b5a13d4150c701abd51efc77da1a43ce77b5714990023428cac3e49d177a8d
+oid sha256:8457bfef7ad4f6da58bb71e46cf268cfe45b3736fc1c15ee5caf831fb0cf880e
 size 828512040

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1e232163e748616f20b19fc434847c2b32b62c446d4d18b0d14156b66cd51537
+oid sha256:a63d79cf6c53dce278f75dc767b2a9a965f6cd8ea3f4dfd8fb9dcbaafddbc097
 size 1657183499

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e337cce6ad62f148d14c3fd6cd72b90800c38358b0b6fcdb66fa80f2f80ad00f
+oid sha256:765906c1a9f94846f3a12888ad5b1f887f418e3827fc791ef4e6d1fd345eb5aa
 size 14709

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6cf85e7928fef52e8e5f5986b3151d88879d53612aa920bb9a39849c5aeac4ad
+oid sha256:491dc3a5c216494ea6d1e4455ab0eadda50a8648129ee0dc968eb77785fc2e3b
 size 1383

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7ef580f26b1f1872d03b95559ecd67d7596de45b58ca2297a19f1f2c1c3aa61
+oid sha256:d9480c4b5829f9309b2468d0cedfc03603c2a007f059a11d6b71534c3a9671d3
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.5439989706639219,
+  "epoch": 1.6983993823983532,
   "eval_steps": 500,
-  "global_step": 100000,
+  "global_step": 110000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -708,6 +708,76 @@
       "learning_rate": 4.560192690722127e-06,
       "loss": 0.011253142356872558,
       "step": 100000
+    },
+    {
+      "epoch": 1.5594390118373649,
+      "grad_norm": 0.001121566048823297,
+      "learning_rate": 4.4057930736331774e-06,
+      "loss": 0.01211655044555664,
+      "step": 101000
+    },
+    {
+      "epoch": 1.5748790530108079,
+      "grad_norm": 0.048229675740003586,
+      "learning_rate": 4.2513934565442285e-06,
+      "loss": 0.01109099006652832,
+      "step": 102000
+    },
+    {
+      "epoch": 1.590319094184251,
+      "grad_norm": 0.013130067847669125,
+      "learning_rate": 4.096993839455279e-06,
+      "loss": 0.011361390113830567,
+      "step": 103000
+    },
+    {
+      "epoch": 1.6057591353576943,
+      "grad_norm": 0.0013874724972993135,
+      "learning_rate": 3.942594222366329e-06,
+      "loss": 0.011926653861999512,
+      "step": 104000
+    },
+    {
+      "epoch": 1.6211991765311375,
+      "grad_norm": 0.2727493643760681,
+      "learning_rate": 3.7881946052773794e-06,
+      "loss": 0.010964672088623046,
+      "step": 105000
+    },
+    {
+      "epoch": 1.6366392177045805,
+      "grad_norm": 0.00750235840678215,
+      "learning_rate": 3.6337949881884295e-06,
+      "loss": 0.012359402656555175,
+      "step": 106000
+    },
+    {
+      "epoch": 1.6520792588780235,
+      "grad_norm": 0.15035952627658844,
+      "learning_rate": 3.4793953710994797e-06,
+      "loss": 0.011547521591186523,
+      "step": 107000
+    },
+    {
+      "epoch": 1.6675193000514668,
+      "grad_norm": 0.24812962114810944,
+      "learning_rate": 3.3249957540105303e-06,
+      "loss": 0.011596759796142579,
+      "step": 108000
+    },
+    {
+      "epoch": 1.68295934122491,
+      "grad_norm": 0.010699417442083359,
+      "learning_rate": 3.170596136921581e-06,
+      "loss": 0.010546285629272461,
+      "step": 109000
+    },
+    {
+      "epoch": 1.6983993823983532,
+      "grad_norm": 0.025276638567447662,
+      "learning_rate": 3.016196519832631e-06,
+      "loss": 0.010572249412536622,
+      "step": 110000
     }
   ],
   "logging_steps": 1000,
@@ -727,7 +797,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.5473777471834112e+18,
+  "total_flos": 1.7018849314330214e+18,
   "train_batch_size": 24,
   "trial_name": null,
   "trial_params": null

run/progress.json

@@ -1,35 +1,35 @@
 {
   "state": "training",
-  "global_step": 100000,
+  "global_step": 110000,
   "max_steps": 129534,
-  "epoch": 1.5439989706639219,
-  "elapsed_seconds": 29312.662812397,
-  "steps_per_second": 2.3880464374050447,
-  "optimizer_updates_this_process": 70000,
+  "epoch": 1.6983993823983532,
+  "elapsed_seconds": 33471.792928825,
+  "steps_per_second": 2.390072147318591,
+  "optimizer_updates_this_process": 80000,
   "peak_cuda_GiB": 16.14784526824951,
   "world_size": 1,
   "effective_batch_size": 72,
   "recent_metrics": [
     {
-      "loss": 0.012590063095092774,
-      "grad_norm": 0.01449698768556118,
-      "learning_rate": 5.023391541988977e-06,
-      "epoch": 1.4976788471435925,
-      "step": 97000
+      "loss": 0.011547521591186523,
+      "grad_norm": 0.15035952627658844,
+      "learning_rate": 3.4793953710994797e-06,
+      "epoch": 1.6520792588780235,
+      "step": 107000
     },
     {
-      "loss": 0.012973206520080566,
-      "grad_norm": 0.07639008015394211,
-      "learning_rate": 4.868991924900027e-06,
-      "epoch": 1.5131188883170354,
-      "step": 98000
+      "loss": 0.011596759796142579,
+      "grad_norm": 0.24812962114810944,
+      "learning_rate": 3.3249957540105303e-06,
+      "epoch": 1.6675193000514668,
+      "step": 108000
     },
     {
-      "loss": 0.011427496910095215,
-      "grad_norm": 0.06926650553941727,
-      "learning_rate": 4.714592307811077e-06,
-      "epoch": 1.5285589294904787,
-      "step": 99000
+      "loss": 0.010546285629272461,
+      "grad_norm": 0.010699417442083359,
+      "learning_rate": 3.170596136921581e-06,
+      "epoch": 1.68295934122491,
+      "step": 109000
     }
   ]
 }