Training in progress, step 1000, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd588dbc728c53e33444470b35f0ec00b32b282a4a741258c3b25ac3abed7937
+oid sha256:2aad20074673be565196c2d84a37e45794bf985e8e924217ecf9324b6beafe71
 size 386379

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2aad02749c94610ad407ffb13a3c15d2904a2ee0a9ac6a2f501b604d4fadc0ef
+oid sha256:2dba33db04b589dba3d8b7f81fc17b5f2c4b85c99cc07e01060cf64fe30716a8
 size 1540661735

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:471c196d464571a2d3eaa8b56640d706baf43a5fa29db23527981ea674d1cc5d
+oid sha256:22df2bd0d59061a382c04efe9789950f253cd4e31ab17e2059fa4c27cc4a6efa
 size 14469

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c4d5d330fd2d21389f18d92d837b7b74326019eda8d0aeef699b67f0b194b16
+oid sha256:5840394b2337e599c4aa005d8e914e87413a0ba6b6dc9fc0e6ef11ae5136e4b7
 size 14469

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f41ef2dc3a9c7dccaf66ef3850e86a97c631081c0fdfdaff5d6a0ce628b0f4f
+oid sha256:b407220d2babdb8285fdb646664c7e9828e3b6748cb46222c7a0af7e23eeea38
 size 14469

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e510800460c42b7b67d24fb27b7099f5c726a948ffebcd7c17501379991ee2ac
+oid sha256:89de852af1b3140eaacd19d758de06a7391d2843e66525fb5f4a916e1aaa0844
 size 14469

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:efbb381c554d799e91db094a8bca7f05b6f679e8066823bcdfc6ea2811eb3c3f
+oid sha256:7035846fe01e1add8b5347f2ac5e39db23b141016ab67fbf3c73bc134ed727a9
 size 14469

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6cd2f2645726a3a2f552a52307dc452892e105e7f362ac69985b42f39311f70b
+oid sha256:b6a21cff1db5937910f9ed43a94ae111bbf1146c9e9881b7d0eb53eb4beac8bd
 size 14469

last-checkpoint/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b974ebc6afa793677214cfcffe57d9f4d7c8e0bd2a082dbc6a3d57bc57e69ab3
+oid sha256:98920b86b1203f91d07132a60121cf9c75c29955c2498a35dcadcff8b0ec4dd8
 size 14469

last-checkpoint/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1df311cc38c26f5641e0477ca47e8af66bb548ffec021086203c4970526e3a77
+oid sha256:57fce693c5b4d0d80e0251092011493286bfb528b7f1955388583bda91701152
 size 14469

last-checkpoint/trainer_state.json

@@ -6,167 +6,167 @@
   "eval_steps": 100,
   "global_step": 1000,
   "is_hyper_param_search": false,
-  "is_local_process_zero": true,
+  "is_local_process_zero": false,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.1,
-      "grad_norm": 1.297243595123291,
+      "grad_norm": 11.13700008392334,
       "learning_rate": 0.0003973800426880847,
-      "loss": 9.179060668945313,
+      "loss": 66.094375,
       "step": 100
     },
     {
       "epoch": 0.1,
-      "eval_accuracy": 0.09053816046966733,
-      "eval_loss": 8.474776268005371,
-      "eval_runtime": 32.6103,
-      "eval_samples_per_second": 30.665,
-      "eval_steps_per_second": 30.665,
+      "eval_accuracy": 0.10251663405088063,
+      "eval_loss": 62.88532638549805,
+      "eval_runtime": 12.9078,
+      "eval_samples_per_second": 9.684,
+      "eval_steps_per_second": 1.24,
       "step": 100
     },
     {
       "epoch": 0.2,
-      "grad_norm": 1.142166256904602,
+      "grad_norm": 6.741835117340088,
       "learning_rate": 0.0003762085737488283,
-      "loss": 8.30135498046875,
+      "loss": 62.0001123046875,
       "step": 200
     },
     {
       "epoch": 0.2,
-      "eval_accuracy": 0.10549706457925637,
-      "eval_loss": 8.138654708862305,
-      "eval_runtime": 26.7248,
-      "eval_samples_per_second": 37.418,
-      "eval_steps_per_second": 37.418,
+      "eval_accuracy": 0.11803913894324854,
+      "eval_loss": 61.526912689208984,
+      "eval_runtime": 8.6535,
+      "eval_samples_per_second": 14.445,
+      "eval_steps_per_second": 1.849,
       "step": 200
     },
     {
       "epoch": 0.3,
-      "grad_norm": 1.286571741104126,
+      "grad_norm": 7.0108513832092285,
       "learning_rate": 0.00033594217168615465,
-      "loss": 7.987710571289062,
+      "loss": 61.543701171875,
       "step": 300
     },
     {
       "epoch": 0.3,
-      "eval_accuracy": 0.11159295499021527,
-      "eval_loss": 7.991443157196045,
-      "eval_runtime": 26.5785,
-      "eval_samples_per_second": 37.624,
-      "eval_steps_per_second": 37.624,
+      "eval_accuracy": 0.11293346379647749,
+      "eval_loss": 60.98686599731445,
+      "eval_runtime": 8.6635,
+      "eval_samples_per_second": 14.428,
+      "eval_steps_per_second": 1.847,
       "step": 300
     },
     {
       "epoch": 0.4,
-      "grad_norm": 1.262372612953186,
+      "grad_norm": 10.79574203491211,
       "learning_rate": 0.00028094432596115747,
-      "loss": 7.810901489257812,
+      "loss": 61.318671875,
       "step": 400
     },
     {
       "epoch": 0.4,
-      "eval_accuracy": 0.10483365949119373,
-      "eval_loss": 7.9381818771362305,
-      "eval_runtime": 26.6986,
-      "eval_samples_per_second": 37.455,
-      "eval_steps_per_second": 37.455,
+      "eval_accuracy": 0.1016477495107632,
+      "eval_loss": 61.48467254638672,
+      "eval_runtime": 8.6224,
+      "eval_samples_per_second": 14.497,
+      "eval_steps_per_second": 1.856,
       "step": 400
     },
     {
       "epoch": 0.5,
-      "grad_norm": 1.2876827716827393,
+      "grad_norm": 6.4634199142456055,
       "learning_rate": 0.00021717490653764342,
-      "loss": 7.900021362304687,
+      "loss": 60.952919921875,
       "step": 500
     },
     {
       "epoch": 0.5,
-      "eval_accuracy": 0.11435420743639922,
-      "eval_loss": 7.851724624633789,
-      "eval_runtime": 26.5522,
-      "eval_samples_per_second": 37.662,
-      "eval_steps_per_second": 37.662,
+      "eval_accuracy": 0.09600978473581213,
+      "eval_loss": 61.699729919433594,
+      "eval_runtime": 8.3835,
+      "eval_samples_per_second": 14.91,
+      "eval_steps_per_second": 1.909,
       "step": 500
     },
     {
       "epoch": 0.6,
-      "grad_norm": 1.2628377676010132,
+      "grad_norm": 9.323127746582031,
       "learning_rate": 0.00015154431949464275,
-      "loss": 7.744283447265625,
+      "loss": 60.478193359375,
       "step": 600
     },
     {
       "epoch": 0.6,
-      "eval_accuracy": 0.1190665362035225,
-      "eval_loss": 7.8012566566467285,
-      "eval_runtime": 27.3957,
-      "eval_samples_per_second": 36.502,
-      "eval_steps_per_second": 36.502,
+      "eval_accuracy": 0.122,
+      "eval_loss": 60.42253494262695,
+      "eval_runtime": 9.026,
+      "eval_samples_per_second": 13.849,
+      "eval_steps_per_second": 1.773,
       "step": 600
     },
     {
       "epoch": 0.7,
-      "grad_norm": 1.3543407917022705,
+      "grad_norm": 9.785552978515625,
       "learning_rate": 9.11646573017482e-05,
-      "loss": 7.696810302734375,
+      "loss": 60.1034619140625,
       "step": 700
     },
     {
       "epoch": 0.7,
-      "eval_accuracy": 0.12185714285714286,
-      "eval_loss": 7.775871276855469,
-      "eval_runtime": 26.9132,
-      "eval_samples_per_second": 37.157,
-      "eval_steps_per_second": 37.157,
+      "eval_accuracy": 0.1186399217221135,
+      "eval_loss": 60.06114196777344,
+      "eval_runtime": 8.5914,
+      "eval_samples_per_second": 14.55,
+      "eval_steps_per_second": 1.862,
       "step": 700
     },
     {
       "epoch": 0.8,
-      "grad_norm": 1.2917762994766235,
+      "grad_norm": 8.054394721984863,
       "learning_rate": 4.2578993244549506e-05,
-      "loss": 7.771747436523437,
+      "loss": 59.673642578125,
       "step": 800
     },
     {
       "epoch": 0.8,
-      "eval_accuracy": 0.12456751467710372,
-      "eval_loss": 7.751345157623291,
-      "eval_runtime": 26.6293,
-      "eval_samples_per_second": 37.553,
-      "eval_steps_per_second": 37.553,
+      "eval_accuracy": 0.13012133072407045,
+      "eval_loss": 59.73853302001953,
+      "eval_runtime": 8.4067,
+      "eval_samples_per_second": 14.869,
+      "eval_steps_per_second": 1.903,
       "step": 800
     },
     {
       "epoch": 0.9,
-      "grad_norm": 1.3219099044799805,
+      "grad_norm": 8.334230422973633,
       "learning_rate": 1.1052337907897503e-05,
-      "loss": 7.84291015625,
+      "loss": 59.854453125,
       "step": 900
     },
     {
       "epoch": 0.9,
-      "eval_accuracy": 0.1239921722113503,
-      "eval_loss": 7.749647617340088,
-      "eval_runtime": 26.5278,
-      "eval_samples_per_second": 37.696,
-      "eval_steps_per_second": 37.696,
+      "eval_accuracy": 0.12184931506849316,
+      "eval_loss": 59.623390197753906,
+      "eval_runtime": 8.5711,
+      "eval_samples_per_second": 14.584,
+      "eval_steps_per_second": 1.867,
       "step": 900
     },
     {
       "epoch": 1.0,
-      "grad_norm": 1.334007978439331,
+      "grad_norm": 8.61923599243164,
       "learning_rate": 1.093583978573065e-09,
-      "loss": 7.778873291015625,
+      "loss": 59.846630859375,
       "step": 1000
     },
     {
       "epoch": 1.0,
-      "eval_accuracy": 0.12396868884540117,
-      "eval_loss": 7.747113227844238,
-      "eval_runtime": 26.6654,
-      "eval_samples_per_second": 37.502,
-      "eval_steps_per_second": 37.502,
+      "eval_accuracy": 0.123146771037182,
+      "eval_loss": 59.59795379638672,
+      "eval_runtime": 8.6863,
+      "eval_samples_per_second": 14.39,
+      "eval_steps_per_second": 1.842,
       "step": 1000
     }
   ],

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b47eb6e798b2dfe4e0fdea550782a8eb56b1b579932ae1b65db9c8ecf145627d
-size 5329
+oid sha256:7eaf4364d37f9f4ceb03f7b16a5643504bb95eb9da30e7cced827582fa3d3389
+size 5265