Training in progress, step 500, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a18eb66379b6698af86fb08013da4c523bb44e653fdb99cabf0ce326b80d22f
-size 386379
+oid sha256:3bade0394fffddee210809f85ef580bee9253cbdc63658c93609636b50849132
+size 2252747

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b1e3a7767d132b2cee73f8c2a62b5f678a31e9a87e52b1878a4eb94b5ebeb94c
-size 1540661735
+oid sha256:abb28c9de07f152f7162a5afb9fa57feec1bb26c13b04027682db0fc06632987
+size 847648963

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:daae0cc70f9e7824a39e0868e7bbd5f4ee69ddcba63b1d42d1220972622fb262
+oid sha256:76e07cf0345bf512c151f0957ea0731d69830b690d0d47fe9335a01fd87ff0d5
 size 14469

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b02a3ca08e76a328358ed359724403d0daf90280ad6996ccb99129eabd7933b
+oid sha256:4b1e066d0ede72061c950708a201039b55678b711860b01a8850b06ca136b2f7
 size 14469

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:881a4302f7ca452bf068ae4fa72a3351c17d2f02cc3ed78ed4d81d0440b092ef
+oid sha256:c83436d7d707c3d61b15981b0298ffc689a42626328497b706511643f5ceb662
 size 14469

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7bbad872a50b1b043373278688384873e9ce6d4b170a2b9147ea2e7045b6f08f
+oid sha256:dcdd3e13e6b97f6783c4798cd70e6e48a2104ccaacf287e70c62bf0c09ec3f22
 size 14469

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f5c33f86cd61c95af37958664568bdef66b045a7dd327e2e4a87418eb5c2617
+oid sha256:aa71eeae497ae80be5331ba680216cbc540af220332f0c56f028ebfc2f445e20
 size 14469

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:745adc34fef116335c62d35da815ab1e6c8444b933918aed8cd7d137cd7d3e67
+oid sha256:c70b31dc63a3cc8a092a19500455139bf70b0744911cd2bfd2067660e23a2f80
 size 14469

last-checkpoint/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9873ce205beb3108aa535f93d98d3355954ff6b888a8d98df447b2c97fc448a
+oid sha256:b86db638d3fa6bb27db28786ecb6ac4913c0fd4a25f7498fd0ae3429244b41eb
 size 14469

last-checkpoint/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:992d5b22958331017a9e6fa5d52716b2c115acc308f2f7b00c82ee11a1c1b070
+oid sha256:840457dd3ad214440eed9722d63d2231facaf5d2730c8917e33dd9d3a6078932
 size 14469

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:94b7b1a014f5dad743694fda5858015bfe3bd4c82eac608d649238b91858a560
+oid sha256:5b79cc0df1e63ecef3adaaf8ef4c455eb58ccb70431c624030057057995b60a8
 size 1465

last-checkpoint/trainer_state.json

@@ -2,179 +2,144 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
-  "eval_steps": 100,
-  "global_step": 1000,
+  "epoch": 0.5,
+  "eval_steps": 50,
+  "global_step": 500,
   "is_hyper_param_search": false,
   "is_local_process_zero": false,
   "is_world_process_zero": true,
   "log_history": [
+    {
+      "epoch": 0.05,
+      "eval_accuracy": 0.07674803149606299,
+      "eval_loss": 59.70803451538086,
+      "eval_runtime": 27.1066,
+      "eval_samples_per_second": 4.611,
+      "eval_steps_per_second": 0.59,
+      "step": 50
+    },
     {
       "epoch": 0.1,
-      "grad_norm": 12.30762004852295,
-      "learning_rate": 0.00039207643405906093,
-      "loss": 216.7740625,
+      "grad_norm": 1437.81298828125,
+      "learning_rate": 0.000264,
+      "loss": 1842.08875,
       "step": 100
     },
     {
       "epoch": 0.1,
-      "eval_accuracy": 0.1816001955034213,
-      "eval_loss": 47.57754135131836,
-      "eval_runtime": 23.1275,
-      "eval_samples_per_second": 5.405,
-      "eval_steps_per_second": 0.173,
+      "eval_accuracy": 0.11214173228346457,
+      "eval_loss": 56.53706359863281,
+      "eval_runtime": 9.6201,
+      "eval_samples_per_second": 12.994,
+      "eval_steps_per_second": 1.663,
       "step": 100
     },
+    {
+      "epoch": 0.15,
+      "eval_accuracy": 0.1248740157480315,
+      "eval_loss": 53.10245132446289,
+      "eval_runtime": 13.0919,
+      "eval_samples_per_second": 9.548,
+      "eval_steps_per_second": 1.222,
+      "step": 150
+    },
     {
       "epoch": 0.2,
-      "grad_norm": 9.95341968536377,
-      "learning_rate": 0.0003650943793925548,
-      "loss": 185.82048828125,
+      "grad_norm": 1170.414794921875,
+      "learning_rate": 0.0003967291041791484,
+      "loss": 1604.2784375,
       "step": 200
     },
     {
       "epoch": 0.2,
-      "eval_accuracy": 0.19564027370478984,
-      "eval_loss": 45.45988082885742,
-      "eval_runtime": 7.2182,
-      "eval_samples_per_second": 17.317,
-      "eval_steps_per_second": 0.554,
+      "eval_accuracy": 0.14473228346456693,
+      "eval_loss": 49.9360237121582,
+      "eval_runtime": 9.3123,
+      "eval_samples_per_second": 13.423,
+      "eval_steps_per_second": 1.718,
       "step": 200
     },
+    {
+      "epoch": 0.25,
+      "eval_accuracy": 0.17015748031496064,
+      "eval_loss": 47.78742218017578,
+      "eval_runtime": 10.4409,
+      "eval_samples_per_second": 11.972,
+      "eval_steps_per_second": 1.532,
+      "step": 250
+    },
     {
       "epoch": 0.3,
-      "grad_norm": 9.293137550354004,
-      "learning_rate": 0.0003216263690654036,
-      "loss": 180.35142578125,
+      "grad_norm": 627.5261840820312,
+      "learning_rate": 0.0003704314025795668,
+      "loss": 1458.88296875,
       "step": 300
     },
     {
       "epoch": 0.3,
-      "eval_accuracy": 0.2023049853372434,
-      "eval_loss": 44.628074645996094,
-      "eval_runtime": 7.037,
-      "eval_samples_per_second": 17.763,
-      "eval_steps_per_second": 0.568,
+      "eval_accuracy": 0.16708661417322834,
+      "eval_loss": 47.620853424072266,
+      "eval_runtime": 9.5812,
+      "eval_samples_per_second": 13.046,
+      "eval_steps_per_second": 1.67,
       "step": 300
     },
+    {
+      "epoch": 0.35,
+      "eval_accuracy": 0.18376377952755907,
+      "eval_loss": 46.354679107666016,
+      "eval_runtime": 8.2523,
+      "eval_samples_per_second": 15.147,
+      "eval_steps_per_second": 1.939,
+      "step": 350
+    },
     {
       "epoch": 0.4,
-      "grad_norm": 9.892265319824219,
-      "learning_rate": 0.00026601302141692667,
-      "loss": 178.04525390625,
+      "grad_norm": 1531.896484375,
+      "learning_rate": 0.0003211159956686115,
+      "loss": 1410.06390625,
       "step": 400
     },
     {
       "epoch": 0.4,
-      "eval_accuracy": 0.20574193548387096,
-      "eval_loss": 44.19523239135742,
-      "eval_runtime": 7.0599,
-      "eval_samples_per_second": 17.706,
-      "eval_steps_per_second": 0.567,
+      "eval_accuracy": 0.18334645669291338,
+      "eval_loss": 45.935264587402344,
+      "eval_runtime": 9.4355,
+      "eval_samples_per_second": 13.248,
+      "eval_steps_per_second": 1.696,
       "step": 400
     },
     {
-      "epoch": 0.5,
-      "grad_norm": 8.574827194213867,
-      "learning_rate": 0.00020380776102066933,
-      "loss": 176.7007421875,
-      "step": 500
+      "epoch": 0.45,
+      "eval_accuracy": 0.18777165354330708,
+      "eval_loss": 45.47114944458008,
+      "eval_runtime": 14.0955,
+      "eval_samples_per_second": 8.868,
+      "eval_steps_per_second": 1.135,
+      "step": 450
     },
     {
       "epoch": 0.5,
-      "eval_accuracy": 0.20855425219941348,
-      "eval_loss": 43.90693283081055,
-      "eval_runtime": 6.876,
-      "eval_samples_per_second": 18.179,
-      "eval_steps_per_second": 0.582,
+      "grad_norm": 19442.306640625,
+      "learning_rate": 0.00025544320241571017,
+      "loss": 1381.1546875,
       "step": 500
     },
     {
-      "epoch": 0.6,
-      "grad_norm": 10.545783042907715,
-      "learning_rate": 0.00014122226612106885,
-      "loss": 175.240859375,
-      "step": 600
-    },
-    {
-      "epoch": 0.6,
-      "eval_accuracy": 0.2102414467253177,
-      "eval_loss": 43.72456741333008,
-      "eval_runtime": 6.9048,
-      "eval_samples_per_second": 18.103,
-      "eval_steps_per_second": 0.579,
-      "step": 600
-    },
-    {
-      "epoch": 0.7,
-      "grad_norm": 10.784721374511719,
-      "learning_rate": 8.450618433005866e-05,
-      "loss": 175.0915234375,
-      "step": 700
-    },
-    {
-      "epoch": 0.7,
-      "eval_accuracy": 0.2116852394916911,
-      "eval_loss": 43.59967803955078,
-      "eval_runtime": 6.9268,
-      "eval_samples_per_second": 18.046,
-      "eval_steps_per_second": 0.577,
-      "step": 700
-    },
-    {
-      "epoch": 0.8,
-      "grad_norm": 7.553999423980713,
-      "learning_rate": 3.932305678751549e-05,
-      "loss": 174.05962890625,
-      "step": 800
-    },
-    {
-      "epoch": 0.8,
-      "eval_accuracy": 0.21238318670576736,
-      "eval_loss": 43.53715515136719,
-      "eval_runtime": 6.7444,
-      "eval_samples_per_second": 18.534,
-      "eval_steps_per_second": 0.593,
-      "step": 800
-    },
-    {
-      "epoch": 0.9,
-      "grad_norm": 8.83249282836914,
-      "learning_rate": 1.0184769603774148e-05,
-      "loss": 174.92396484375,
-      "step": 900
-    },
-    {
-      "epoch": 0.9,
-      "eval_accuracy": 0.21264418377321603,
-      "eval_loss": 43.506919860839844,
-      "eval_runtime": 6.9005,
-      "eval_samples_per_second": 18.115,
-      "eval_steps_per_second": 0.58,
-      "step": 900
-    },
-    {
-      "epoch": 1.0,
-      "grad_norm": 8.517117500305176,
-      "learning_rate": 1.0069988897853933e-09,
-      "loss": 173.867734375,
-      "step": 1000
-    },
-    {
-      "epoch": 1.0,
-      "eval_accuracy": 0.21278690127077224,
-      "eval_loss": 43.502532958984375,
-      "eval_runtime": 6.8719,
-      "eval_samples_per_second": 18.19,
-      "eval_steps_per_second": 0.582,
-      "step": 1000
+      "epoch": 0.5,
+      "eval_accuracy": 0.18888188976377954,
+      "eval_loss": 45.3370475769043,
+      "eval_runtime": 9.3555,
+      "eval_samples_per_second": 13.361,
+      "eval_steps_per_second": 1.71,
+      "step": 500
     }
   ],
   "logging_steps": 100,
   "max_steps": 1000,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 9223372036854775807,
-  "save_steps": 2000,
+  "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -182,13 +147,13 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 4,
+  "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e42ac3e5a09bb56fc92faab3744d893d5a5809db1bd4cadabb4c3a55653610d2
+oid sha256:b4104560ef5b6e6db26b36daa57a06deeead0a170fcb81c70f3e9dbd8207e3ca
 size 5265