Model save

all_results.json

@@ -1,8 +1,8 @@
 {
-    "total_flos": 5.211391058037965e+16,
-    "train_loss": 0.09458173938046872,
-    "train_runtime": 117.3732,
-    "train_samples": 1000,
-    "train_samples_per_second": 164.066,
-    "train_steps_per_second": 1.304
+    "total_flos": 3.3879319029743616e+16,
+    "train_loss": 0.10813453071045154,
+    "train_runtime": 80.1681,
+    "train_samples": 1086,
+    "train_samples_per_second": 156.159,
+    "train_steps_per_second": 1.235
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "total_flos": 5.211391058037965e+16,
-    "train_loss": 0.09458173938046872,
-    "train_runtime": 117.3732,
-    "train_samples": 1000,
-    "train_samples_per_second": 164.066,
-    "train_steps_per_second": 1.304
+    "total_flos": 3.3879319029743616e+16,
+    "train_loss": 0.10813453071045154,
+    "train_runtime": 80.1681,
+    "train_samples": 1086,
+    "train_samples_per_second": 156.159,
+    "train_steps_per_second": 1.235
 }

trainer_state.json

@@ -4,236 +4,159 @@
   "best_model_checkpoint": null,
   "epoch": 3.0,
   "eval_steps": 500,
-  "global_step": 153,
+  "global_step": 99,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.09803921568627451,
-      "grad_norm": 0.375,
-      "learning_rate": 2.5e-05,
-      "loss": 0.1074,
+      "epoch": 0.15151515151515152,
+      "grad_norm": 0.4375,
+      "learning_rate": 2.4e-05,
+      "loss": 0.1336,
       "step": 5
     },
     {
-      "epoch": 0.19607843137254902,
-      "grad_norm": 0.3359375,
-      "learning_rate": 4.9994719205415894e-05,
-      "loss": 0.099,
+      "epoch": 0.30303030303030304,
+      "grad_norm": 0.466796875,
+      "learning_rate": 2.9879546090089533e-05,
+      "loss": 0.1225,
       "step": 10
     },
     {
-      "epoch": 0.29411764705882354,
-      "grad_norm": 0.318359375,
-      "learning_rate": 4.981015154140181e-05,
-      "loss": 0.1039,
+      "epoch": 0.45454545454545453,
+      "grad_norm": 0.333984375,
+      "learning_rate": 2.9393883712293316e-05,
+      "loss": 0.1116,
       "step": 15
     },
     {
-      "epoch": 0.39215686274509803,
-      "grad_norm": 0.298828125,
-      "learning_rate": 4.936401774893088e-05,
-      "loss": 0.1032,
+      "epoch": 0.6060606060606061,
+      "grad_norm": 0.31640625,
+      "learning_rate": 2.8549004284108398e-05,
+      "loss": 0.1105,
       "step": 20
     },
     {
-      "epoch": 0.49019607843137253,
-      "grad_norm": 0.302734375,
-      "learning_rate": 4.8661548338815794e-05,
-      "loss": 0.1,
+      "epoch": 0.7575757575757576,
+      "grad_norm": 0.29296875,
+      "learning_rate": 2.7368445717222102e-05,
+      "loss": 0.1082,
       "step": 25
     },
     {
-      "epoch": 0.5882352941176471,
-      "grad_norm": 0.267578125,
-      "learning_rate": 4.771097912261625e-05,
-      "loss": 0.0995,
+      "epoch": 0.9090909090909091,
+      "grad_norm": 0.271484375,
+      "learning_rate": 2.5885097773607675e-05,
+      "loss": 0.1098,
       "step": 30
     },
     {
-      "epoch": 0.6862745098039216,
-      "grad_norm": 0.287109375,
-      "learning_rate": 4.652345465527847e-05,
-      "loss": 0.1007,
+      "epoch": 1.0606060606060606,
+      "grad_norm": 0.26953125,
+      "learning_rate": 2.4140285773463036e-05,
+      "loss": 0.1056,
       "step": 35
     },
     {
-      "epoch": 0.7843137254901961,
-      "grad_norm": 0.27734375,
-      "learning_rate": 4.511289757541801e-05,
-      "loss": 0.098,
+      "epoch": 1.2121212121212122,
+      "grad_norm": 0.267578125,
+      "learning_rate": 2.2182619292782524e-05,
+      "loss": 0.1084,
       "step": 40
     },
     {
-      "epoch": 0.8823529411764706,
-      "grad_norm": 0.271484375,
-      "learning_rate": 4.349584537511159e-05,
-      "loss": 0.0968,
+      "epoch": 1.3636363636363638,
+      "grad_norm": 0.28125,
+      "learning_rate": 2.0066637925262362e-05,
+      "loss": 0.1048,
       "step": 45
     },
     {
-      "epoch": 0.9803921568627451,
-      "grad_norm": 0.2578125,
-      "learning_rate": 4.16912565129229e-05,
-      "loss": 0.0988,
+      "epoch": 1.5151515151515151,
+      "grad_norm": 0.26953125,
+      "learning_rate": 1.7851291836925332e-05,
+      "loss": 0.1057,
       "step": 50
     },
     {
-      "epoch": 1.0784313725490196,
-      "grad_norm": 0.279296875,
-      "learning_rate": 3.9720288143309735e-05,
-      "loss": 0.0961,
+      "epoch": 1.6666666666666665,
+      "grad_norm": 0.2734375,
+      "learning_rate": 1.559829944444086e-05,
+      "loss": 0.1046,
       "step": 55
     },
     {
-      "epoch": 1.1764705882352942,
-      "grad_norm": 0.271484375,
-      "learning_rate": 3.7606048068332286e-05,
-      "loss": 0.0909,
+      "epoch": 1.8181818181818183,
+      "grad_norm": 0.259765625,
+      "learning_rate": 1.3370427971388369e-05,
+      "loss": 0.1057,
       "step": 60
     },
     {
-      "epoch": 1.2745098039215685,
-      "grad_norm": 0.30078125,
-      "learning_rate": 3.5373323819801494e-05,
-      "loss": 0.0942,
+      "epoch": 1.9696969696969697,
+      "grad_norm": 0.287109375,
+      "learning_rate": 1.1229744785292821e-05,
+      "loss": 0.1048,
       "step": 65
     },
     {
-      "epoch": 1.3725490196078431,
-      "grad_norm": 0.267578125,
-      "learning_rate": 3.304829204813215e-05,
-      "loss": 0.09,
+      "epoch": 2.121212121212121,
+      "grad_norm": 0.25390625,
+      "learning_rate": 9.235888232294472e-06,
+      "loss": 0.104,
       "step": 70
     },
     {
-      "epoch": 1.4705882352941178,
-      "grad_norm": 0.271484375,
-      "learning_rate": 3.065821162505025e-05,
-      "loss": 0.0932,
+      "epoch": 2.2727272727272725,
+      "grad_norm": 0.26171875,
+      "learning_rate": 7.444406143120487e-06,
+      "loss": 0.1019,
       "step": 75
     },
     {
-      "epoch": 1.5686274509803921,
-      "grad_norm": 0.255859375,
-      "learning_rate": 2.8231104058245068e-05,
-      "loss": 0.0894,
+      "epoch": 2.4242424242424243,
+      "grad_norm": 0.271484375,
+      "learning_rate": 5.9052082987380775e-06,
+      "loss": 0.1052,
       "step": 80
     },
     {
-      "epoch": 1.6666666666666665,
-      "grad_norm": 0.25390625,
-      "learning_rate": 2.579542496481177e-05,
-      "loss": 0.0964,
+      "epoch": 2.5757575757575757,
+      "grad_norm": 0.2578125,
+      "learning_rate": 4.6611759692099345e-06,
+      "loss": 0.1047,
       "step": 85
     },
     {
-      "epoch": 1.7647058823529411,
-      "grad_norm": 0.29296875,
-      "learning_rate": 2.3379730455158238e-05,
-      "loss": 0.0897,
+      "epoch": 2.7272727272727275,
+      "grad_norm": 0.2578125,
+      "learning_rate": 3.746967263293098e-06,
+      "loss": 0.1048,
       "step": 90
     },
     {
-      "epoch": 1.8627450980392157,
-      "grad_norm": 0.26171875,
-      "learning_rate": 2.101234233871961e-05,
-      "loss": 0.0937,
+      "epoch": 2.878787878787879,
+      "grad_norm": 0.26953125,
+      "learning_rate": 3.188051571134615e-06,
+      "loss": 0.1035,
       "step": 95
     },
-    {
-      "epoch": 1.9607843137254903,
-      "grad_norm": 0.279296875,
-      "learning_rate": 1.8721016076637528e-05,
-      "loss": 0.0909,
-      "step": 100
-    },
-    {
-      "epoch": 2.0588235294117645,
-      "grad_norm": 0.279296875,
-      "learning_rate": 1.6532615374355324e-05,
-      "loss": 0.0966,
-      "step": 105
-    },
-    {
-      "epoch": 2.156862745098039,
-      "grad_norm": 0.24609375,
-      "learning_rate": 1.4472797229233409e-05,
-      "loss": 0.0882,
-      "step": 110
-    },
-    {
-      "epoch": 2.2549019607843137,
-      "grad_norm": 0.251953125,
-      "learning_rate": 1.2565711125713656e-05,
-      "loss": 0.0869,
-      "step": 115
-    },
-    {
-      "epoch": 2.3529411764705883,
-      "grad_norm": 0.2578125,
-      "learning_rate": 1.0833715904694373e-05,
-      "loss": 0.0909,
-      "step": 120
-    },
-    {
-      "epoch": 2.450980392156863,
-      "grad_norm": 0.26171875,
-      "learning_rate": 9.297117626563687e-06,
-      "loss": 0.0902,
-      "step": 125
-    },
-    {
-      "epoch": 2.549019607843137,
-      "grad_norm": 0.23046875,
-      "learning_rate": 7.973931501207469e-06,
-      "loss": 0.091,
-      "step": 130
-    },
-    {
-      "epoch": 2.6470588235294117,
-      "grad_norm": 0.283203125,
-      "learning_rate": 6.879670676144916e-06,
-      "loss": 0.0914,
-      "step": 135
-    },
-    {
-      "epoch": 2.7450980392156863,
-      "grad_norm": 0.240234375,
-      "learning_rate": 6.027164359057668e-06,
-      "loss": 0.0909,
-      "step": 140
-    },
-    {
-      "epoch": 2.843137254901961,
-      "grad_norm": 0.2333984375,
-      "learning_rate": 5.426407407059619e-06,
-      "loss": 0.0888,
-      "step": 145
-    },
-    {
-      "epoch": 2.9411764705882355,
-      "grad_norm": 0.2431640625,
-      "learning_rate": 5.084443146135623e-06,
-      "loss": 0.0911,
-      "step": 150
-    },
     {
       "epoch": 3.0,
-      "step": 153,
-      "total_flos": 5.211391058037965e+16,
-      "train_loss": 0.09458173938046872,
-      "train_runtime": 117.3732,
-      "train_samples_per_second": 164.066,
-      "train_steps_per_second": 1.304
+      "step": 99,
+      "total_flos": 3.3879319029743616e+16,
+      "train_loss": 0.10813453071045154,
+      "train_runtime": 80.1681,
+      "train_samples_per_second": 156.159,
+      "train_steps_per_second": 1.235
     }
   ],
   "logging_steps": 5,
-  "max_steps": 153,
+  "max_steps": 99,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
-  "save_steps": 100,
+  "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -246,7 +169,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 5.211391058037965e+16,
+  "total_flos": 3.3879319029743616e+16,
   "train_batch_size": 128,
   "trial_name": null,
   "trial_params": null