Checkpoint at step 70000

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d3e992a0f30cb85e222adf7746af890706c1928ac935c3b0db931e7c170777dd
+oid sha256:f1ad80b1bdb44142d4f811ddeae25e9134e3aa56a3e1e11943ea2f8bba19f093
 size 828512040

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:edd904294400a12350bffa24abd2b4df9d0d80bf11fbb29648a2553be74343ab
+oid sha256:71bfcc7d8776895ab74bd3a67ce9485088e79cf505c1753b5a057ae55a2cefcb
 size 1657183499

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:156c466852e1f42ddaf500b1bed352ad6efa73e3badab28f5a69569e142e7bd0
+oid sha256:8e727a9485655a71ec5a38405bbde585e420d293efe8eda2ba5c7e0aa2b10acd
 size 14709

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc5f5684022d80a5c1634c644a56a208c802d202d5582512fb546d082a45700b
+oid sha256:f9f1e5d4e52a00a37ecea9f307a4b3f0e21ccf198af1e615ad26e055bd9fa46d
 size 1383

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b15ee788e6b121840594d0284d766c854fcb6eebbe89d11447029f6c8584ee40
+oid sha256:4f3cdaf022b77ca7929b10c6704de01ca7b1e757d0abc53fd76e85b2280b0127
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9264024704065877,
+  "epoch": 1.080797735460628,
   "eval_steps": 500,
-  "global_step": 60000,
+  "global_step": 70000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -428,6 +428,76 @@
       "learning_rate": 1.0736177374280113e-05,
       "loss": 0.01389419174194336,
       "step": 60000
+    },
+    {
+      "epoch": 0.9418425115800309,
+      "grad_norm": 0.04992588981986046,
+      "learning_rate": 1.0581777757191162e-05,
+      "loss": 0.014293676376342773,
+      "step": 61000
+    },
+    {
+      "epoch": 0.957282552753474,
+      "grad_norm": 1.4608455896377563,
+      "learning_rate": 1.0427378140102213e-05,
+      "loss": 0.01511683750152588,
+      "step": 62000
+    },
+    {
+      "epoch": 0.9727225939269172,
+      "grad_norm": 0.3881261944770813,
+      "learning_rate": 1.0272978523013264e-05,
+      "loss": 0.015200254440307617,
+      "step": 63000
+    },
+    {
+      "epoch": 0.9881626351003603,
+      "grad_norm": 0.020133264362812042,
+      "learning_rate": 1.0118578905924314e-05,
+      "loss": 0.013801728248596191,
+      "step": 64000
+    },
+    {
+      "epoch": 1.0035975295934123,
+      "grad_norm": 0.35319793224334717,
+      "learning_rate": 9.964179288835365e-06,
+      "loss": 0.01431869125366211,
+      "step": 65000
+    },
+    {
+      "epoch": 1.0190375707668553,
+      "grad_norm": 0.02204095385968685,
+      "learning_rate": 9.809779671746414e-06,
+      "loss": 0.013113049507141114,
+      "step": 66000
+    },
+    {
+      "epoch": 1.0344776119402985,
+      "grad_norm": 0.010962982662022114,
+      "learning_rate": 9.655380054657465e-06,
+      "loss": 0.013373047828674316,
+      "step": 67000
+    },
+    {
+      "epoch": 1.0499176531137415,
+      "grad_norm": 0.23546403646469116,
+      "learning_rate": 9.500980437568516e-06,
+      "loss": 0.011690358161926269,
+      "step": 68000
+    },
+    {
+      "epoch": 1.0653576942871847,
+      "grad_norm": 0.1257835030555725,
+      "learning_rate": 9.346580820479565e-06,
+      "loss": 0.012628182411193848,
+      "step": 69000
+    },
+    {
+      "epoch": 1.080797735460628,
+      "grad_norm": 0.057164762169122696,
+      "learning_rate": 9.192181203390616e-06,
+      "loss": 0.013737136840820313,
+      "step": 70000
     }
   ],
   "logging_steps": 1000,
@@ -447,7 +517,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 9.279923376395059e+17,
+  "total_flos": 1.0825259449525862e+18,
   "train_batch_size": 24,
   "trial_name": null,
   "trial_params": null

run/progress.json

@@ -1,35 +1,35 @@
 {
   "state": "training",
-  "global_step": 60000,
+  "global_step": 70000,
   "max_steps": 129534,
-  "epoch": 0.9264024704065877,
-  "elapsed_seconds": 12119.297969527004,
-  "steps_per_second": 2.475390907578358,
-  "optimizer_updates_this_process": 30000,
+  "epoch": 1.080797735460628,
+  "elapsed_seconds": 16431.000847582996,
+  "steps_per_second": 2.43442261193018,
+  "optimizer_updates_this_process": 40000,
   "peak_cuda_GiB": 16.147835731506348,
   "world_size": 1,
   "effective_batch_size": 72,
   "recent_metrics": [
     {
-      "loss": 0.014366375923156737,
-      "grad_norm": 0.23493172228336334,
-      "learning_rate": 1.119937622554696e-05,
-      "epoch": 0.8800823468862584,
-      "step": 57000
+      "loss": 0.013373047828674316,
+      "grad_norm": 0.010962982662022114,
+      "learning_rate": 9.655380054657465e-06,
+      "epoch": 1.0344776119402985,
+      "step": 67000
     },
     {
-      "loss": 0.014213233947753907,
-      "grad_norm": 0.008305946364998817,
-      "learning_rate": 1.1044976608458011e-05,
-      "epoch": 0.8955223880597015,
-      "step": 58000
+      "loss": 0.011690358161926269,
+      "grad_norm": 0.23546403646469116,
+      "learning_rate": 9.500980437568516e-06,
+      "epoch": 1.0499176531137415,
+      "step": 68000
     },
     {
-      "loss": 0.014378649711608887,
-      "grad_norm": 0.003681991482153535,
-      "learning_rate": 1.0890576991369062e-05,
-      "epoch": 0.9109624292331446,
-      "step": 59000
+      "loss": 0.012628182411193848,
+      "grad_norm": 0.1257835030555725,
+      "learning_rate": 9.346580820479565e-06,
+      "epoch": 1.0653576942871847,
+      "step": 69000
     }
   ]
 }