Checkpoint at step 60000

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bed344f4e6ec0ab4f3b67a3c9cc2fbf1862edaa4c9779ef7eec194aad98c31ee
+oid sha256:d3e992a0f30cb85e222adf7746af890706c1928ac935c3b0db931e7c170777dd
 size 828512040

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f4055448e576b241a37c2bf8f91a66d9e3fd12006d1b596faa06c7c44bd5dd6
+oid sha256:edd904294400a12350bffa24abd2b4df9d0d80bf11fbb29648a2553be74343ab
 size 1657183499

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:12ece05b2ede9df156262979286e339c055e8c2fe8447054b30a9471fde11074
+oid sha256:156c466852e1f42ddaf500b1bed352ad6efa73e3badab28f5a69569e142e7bd0
 size 14709

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d104a96a640ef80c3c92be9df98931218ac558e8bc30b38f25df2a21f8fb58ea
+oid sha256:fc5f5684022d80a5c1634c644a56a208c802d202d5582512fb546d082a45700b
 size 1383

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ac48ccb0ca9cc2f554de5934a471ee333c535d947c0271e83c358c05bdf5522
+oid sha256:b15ee788e6b121840594d0284d766c854fcb6eebbe89d11447029f6c8584ee40
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.7720020586721564,
+  "epoch": 0.9264024704065877,
   "eval_steps": 500,
-  "global_step": 50000,
+  "global_step": 60000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -358,6 +358,76 @@
       "learning_rate": 1.228017354516961e-05,
       "loss": 0.015569474220275879,
       "step": 50000
+    },
+    {
+      "epoch": 0.7874420998455995,
+      "grad_norm": NaN,
+      "learning_rate": 1.2125773928080659e-05,
+      "loss": 0.016073102951049804,
+      "step": 51000
+    },
+    {
+      "epoch": 0.8028821410190428,
+      "grad_norm": 0.18823210895061493,
+      "learning_rate": 1.197137431099171e-05,
+      "loss": 0.015385358810424805,
+      "step": 52000
+    },
+    {
+      "epoch": 0.8183221821924859,
+      "grad_norm": 0.009227069094777107,
+      "learning_rate": 1.1816974693902761e-05,
+      "loss": 0.01468699836730957,
+      "step": 53000
+    },
+    {
+      "epoch": 0.833762223365929,
+      "grad_norm": 0.2508651614189148,
+      "learning_rate": 1.1662575076813812e-05,
+      "loss": 0.01593993663787842,
+      "step": 54000
+    },
+    {
+      "epoch": 0.8492022645393721,
+      "grad_norm": 0.001811643480323255,
+      "learning_rate": 1.1508175459724861e-05,
+      "loss": 0.015393324851989746,
+      "step": 55000
+    },
+    {
+      "epoch": 0.8646423057128152,
+      "grad_norm": 0.14487652480602264,
+      "learning_rate": 1.135377584263591e-05,
+      "loss": 0.013592599868774414,
+      "step": 56000
+    },
+    {
+      "epoch": 0.8800823468862584,
+      "grad_norm": 0.23493172228336334,
+      "learning_rate": 1.119937622554696e-05,
+      "loss": 0.014366375923156737,
+      "step": 57000
+    },
+    {
+      "epoch": 0.8955223880597015,
+      "grad_norm": 0.008305946364998817,
+      "learning_rate": 1.1044976608458011e-05,
+      "loss": 0.014213233947753907,
+      "step": 58000
+    },
+    {
+      "epoch": 0.9109624292331446,
+      "grad_norm": 0.003681991482153535,
+      "learning_rate": 1.0890576991369062e-05,
+      "loss": 0.014378649711608887,
+      "step": 59000
+    },
+    {
+      "epoch": 0.9264024704065877,
+      "grad_norm": 0.22534188628196716,
+      "learning_rate": 1.0736177374280113e-05,
+      "loss": 0.01389419174194336,
+      "step": 60000
     }
   ],
   "logging_steps": 1000,
@@ -377,7 +447,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 7.730943166037606e+17,
+  "total_flos": 9.279923376395059e+17,
   "train_batch_size": 24,
   "trial_name": null,
   "trial_params": null

run/progress.json

@@ -1,35 +1,35 @@
 {
   "state": "training",
-  "global_step": 50000,
+  "global_step": 60000,
   "max_steps": 129534,
-  "epoch": 0.7720020586721564,
-  "elapsed_seconds": 7970.391427626004,
-  "steps_per_second": 2.509287050906738,
-  "optimizer_updates_this_process": 20000,
+  "epoch": 0.9264024704065877,
+  "elapsed_seconds": 12119.297969527004,
+  "steps_per_second": 2.475390907578358,
+  "optimizer_updates_this_process": 30000,
   "peak_cuda_GiB": 16.147835731506348,
   "world_size": 1,
   "effective_batch_size": 72,
   "recent_metrics": [
     {
-      "loss": 0.01701980781555176,
-      "grad_norm": 0.016359159722924232,
-      "learning_rate": 1.2743372396436458e-05,
-      "epoch": 0.7256819351518271,
-      "step": 47000
+      "loss": 0.014366375923156737,
+      "grad_norm": 0.23493172228336334,
+      "learning_rate": 1.119937622554696e-05,
+      "epoch": 0.8800823468862584,
+      "step": 57000
     },
     {
-      "loss": 0.017957780838012697,
-      "grad_norm": 0.35645583271980286,
-      "learning_rate": 1.2588972779347508e-05,
-      "epoch": 0.7411219763252702,
-      "step": 48000
+      "loss": 0.014213233947753907,
+      "grad_norm": 0.008305946364998817,
+      "learning_rate": 1.1044976608458011e-05,
+      "epoch": 0.8955223880597015,
+      "step": 58000
     },
     {
-      "loss": 0.016010673522949218,
-      "grad_norm": 0.5092584490776062,
-      "learning_rate": 1.2434573162258559e-05,
-      "epoch": 0.7565620174987133,
-      "step": 49000
+      "loss": 0.014378649711608887,
+      "grad_norm": 0.003681991482153535,
+      "learning_rate": 1.0890576991369062e-05,
+      "epoch": 0.9109624292331446,
+      "step": 59000
     }
   ]
 }