Training in progress, step 1000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f3c22936a0095ac18daff8e6dcf0e5a3c013ec81937d069c3fc7ec2ce742258
+oid sha256:d4ceb1eb66e59b2be3b7b56c92ceeee0dff3d3afb66f4e798a3c7f7a973769da
 size 150625560

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ede7891a612e5bdf9d9f3ea5ba8ce2d972ac251ac865d5a7ca14b4f136bc03db
+oid sha256:300510ba8259c34ff91864564df369f7ef5da88e12e952b5fc750f0733f75299
 size 602335994

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:521f432674dde45244a4689a033d9db107314cdff516b6bf8384096ef5f6c190
+oid sha256:dec81976f45ae46b24f8606fae652d67e12c656a1fb7001a6621b0275456c831
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e1c35f90edf203b2f0483c8d4d666d73bb8fd60a06a785c4abd30a947f264d74
+oid sha256:ebbd97a5f85f91a146bd613ec7c9e8b4ff3314b259cf17fa7c45467f21162760
 size 1064

last-checkpoint/trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.24140740517215364,
+  "epoch": 0.24141104757306456,
   "eval_steps": 500,
   "global_step": 1000,
   "is_hyper_param_search": false,
@@ -10,228 +10,228 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.012070370258607683,
-      "grad_norm": 1.453125,
-      "learning_rate": 1.5087507543753772e-06,
-      "loss": 2.681,
-      "mean_token_accuracy": 0.5007504414767027,
-      "num_input_tokens_seen": 25446880,
+      "epoch": 0.012070552378653229,
+      "grad_norm": 1.5390625,
+      "learning_rate": 7.543753771876886e-07,
+      "loss": 2.6793,
+      "mean_token_accuracy": 0.5008124655112625,
+      "num_input_tokens_seen": 24135392,
       "num_tokens": 10166805.0,
       "step": 50
     },
     {
-      "epoch": 0.024140740517215365,
-      "grad_norm": 1.4921875,
-      "learning_rate": 3.0175015087507544e-06,
-      "loss": 2.6838,
-      "mean_token_accuracy": 0.500230128839612,
-      "num_input_tokens_seen": 51020960,
+      "epoch": 0.024141104757306457,
+      "grad_norm": 1.578125,
+      "learning_rate": 1.5087507543753772e-06,
+      "loss": 2.683,
+      "mean_token_accuracy": 0.5005724773928523,
+      "num_input_tokens_seen": 48153152,
       "num_tokens": 20365412.0,
       "step": 100
     },
     {
-      "epoch": 0.03621111077582305,
-      "grad_norm": 1.6484375,
-      "learning_rate": 4.526252263126131e-06,
-      "loss": 2.6811,
-      "mean_token_accuracy": 0.5007199160754681,
-      "num_input_tokens_seen": 76507776,
+      "epoch": 0.03621165713595968,
+      "grad_norm": 4.25,
+      "learning_rate": 2.2631261315630656e-06,
+      "loss": 2.6827,
+      "mean_token_accuracy": 0.5006869393214584,
+      "num_input_tokens_seen": 72337344,
       "num_tokens": 30424121.0,
       "step": 150
     },
     {
-      "epoch": 0.04828148103443073,
-      "grad_norm": 1.4765625,
-      "learning_rate": 6.035003017501509e-06,
-      "loss": 2.6797,
-      "mean_token_accuracy": 0.500353229418397,
-      "num_input_tokens_seen": 101977920,
+      "epoch": 0.048282209514612914,
+      "grad_norm": 1.6640625,
+      "learning_rate": 3.0175015087507544e-06,
+      "loss": 2.6854,
+      "mean_token_accuracy": 0.49996432337909935,
+      "num_input_tokens_seen": 96540720,
       "num_tokens": 40650709.0,
       "step": 200
     },
     {
-      "epoch": 0.06035185129303841,
-      "grad_norm": 1.421875,
-      "learning_rate": 7.543753771876886e-06,
-      "loss": 2.6464,
-      "mean_token_accuracy": 0.5040991773456335,
-      "num_input_tokens_seen": 127536896,
+      "epoch": 0.06035276189326614,
+      "grad_norm": 1.671875,
+      "learning_rate": 3.771876885938443e-06,
+      "loss": 2.6589,
+      "mean_token_accuracy": 0.5033125403895974,
+      "num_input_tokens_seen": 120664016,
       "num_tokens": 50830609.0,
       "step": 250
     },
     {
-      "epoch": 0.0724222215516461,
-      "grad_norm": 1.140625,
-      "learning_rate": 9.052504526252262e-06,
-      "loss": 2.654,
-      "mean_token_accuracy": 0.5034311909228564,
-      "num_input_tokens_seen": 153048544,
+      "epoch": 0.07242331427191936,
+      "grad_norm": 1.4375,
+      "learning_rate": 4.526252263126131e-06,
+      "loss": 2.6724,
+      "mean_token_accuracy": 0.5016581188514828,
+      "num_input_tokens_seen": 144779584,
       "num_tokens": 60898431.0,
       "step": 300
     },
     {
-      "epoch": 0.08449259181025377,
-      "grad_norm": 1.0234375,
-      "learning_rate": 1.0561255280627642e-05,
-      "loss": 2.625,
-      "mean_token_accuracy": 0.5059863431751728,
-      "num_input_tokens_seen": 178595392,
+      "epoch": 0.0844938666505726,
+      "grad_norm": 1.484375,
+      "learning_rate": 5.280627640313821e-06,
+      "loss": 2.6515,
+      "mean_token_accuracy": 0.5036601684615016,
+      "num_input_tokens_seen": 168931312,
       "num_tokens": 71195944.0,
       "step": 350
     },
     {
-      "epoch": 0.09656296206886146,
-      "grad_norm": 0.91015625,
-      "learning_rate": 1.2070006035003018e-05,
-      "loss": 2.6079,
-      "mean_token_accuracy": 0.5077864757180214,
-      "num_input_tokens_seen": 204085184,
+      "epoch": 0.09656441902922583,
+      "grad_norm": 1.40625,
+      "learning_rate": 6.035003017501509e-06,
+      "loss": 2.6474,
+      "mean_token_accuracy": 0.5041869094967842,
+      "num_input_tokens_seen": 192934592,
       "num_tokens": 81381172.0,
       "step": 400
     },
     {
-      "epoch": 0.10863333232746915,
-      "grad_norm": 0.78515625,
-      "learning_rate": 1.3578756789378394e-05,
-      "loss": 2.6083,
-      "mean_token_accuracy": 0.5078243865072727,
-      "num_input_tokens_seen": 229609824,
+      "epoch": 0.10863497140787905,
+      "grad_norm": 1.3359375,
+      "learning_rate": 6.789378394689197e-06,
+      "loss": 2.6561,
+      "mean_token_accuracy": 0.5037067172303796,
+      "num_input_tokens_seen": 217010448,
       "num_tokens": 91569720.0,
       "step": 450
     },
     {
-      "epoch": 0.12070370258607682,
-      "grad_norm": 0.734375,
-      "learning_rate": 1.5087507543753773e-05,
-      "loss": 2.591,
-      "num_input_tokens_seen": 255055520,
+      "epoch": 0.12070552378653228,
+      "grad_norm": 1.2421875,
+      "learning_rate": 7.543753771876886e-06,
+      "loss": 2.6454,
+      "num_input_tokens_seen": 241172672,
       "step": 500
     },
     {
-      "epoch": 0.12070370258607682,
-      "eval_loss": 2.492645263671875,
-      "eval_mean_token_accuracy": 0.5286665781386491,
+      "epoch": 0.12070552378653228,
+      "eval_loss": 2.550887107849121,
+      "eval_mean_token_accuracy": 0.5239145074365208,
       "eval_num_tokens": 101801812.0,
-      "eval_runtime": 126.4646,
-      "eval_samples_per_second": 84.704,
-      "eval_steps_per_second": 21.176,
-      "num_input_tokens_seen": 255055520,
+      "eval_runtime": 125.6328,
+      "eval_samples_per_second": 85.264,
+      "eval_steps_per_second": 21.316,
+      "num_input_tokens_seen": 241172672,
       "step": 500
     },
     {
-      "epoch": 0.1327740728446845,
-      "grad_norm": 0.70703125,
-      "learning_rate": 1.659625829812915e-05,
-      "loss": 2.5826,
-      "mean_token_accuracy": 0.5089771485328675,
-      "num_input_tokens_seen": 280435680,
+      "epoch": 0.1327760761651855,
+      "grad_norm": 1.234375,
+      "learning_rate": 8.298129149064575e-06,
+      "loss": 2.6413,
+      "mean_token_accuracy": 0.5040716730989516,
+      "num_input_tokens_seen": 265089232,
       "num_tokens": 111840270.0,
       "step": 550
     },
     {
-      "epoch": 0.1448444431032922,
-      "grad_norm": 0.65625,
-      "learning_rate": 1.8105009052504525e-05,
-      "loss": 2.5689,
-      "mean_token_accuracy": 0.5112711164355278,
-      "num_input_tokens_seen": 305931936,
+      "epoch": 0.14484662854383873,
+      "grad_norm": 1.0859375,
+      "learning_rate": 9.052504526252262e-06,
+      "loss": 2.6279,
+      "mean_token_accuracy": 0.5059382322058081,
+      "num_input_tokens_seen": 289296624,
       "num_tokens": 121987982.0,
       "step": 600
     },
     {
-      "epoch": 0.15691481336189989,
-      "grad_norm": 0.75,
-      "learning_rate": 1.9613759806879906e-05,
-      "loss": 2.55,
-      "mean_token_accuracy": 0.5135074812173843,
-      "num_input_tokens_seen": 331404160,
+      "epoch": 0.15691718092249196,
+      "grad_norm": 0.93359375,
+      "learning_rate": 9.806879903439953e-06,
+      "loss": 2.6104,
+      "mean_token_accuracy": 0.5081973234936595,
+      "num_input_tokens_seen": 313326128,
       "num_tokens": 132144416.0,
       "step": 650
     },
     {
-      "epoch": 0.16898518362050755,
-      "grad_norm": 0.58984375,
-      "learning_rate": 2.1122510561255283e-05,
-      "loss": 2.5497,
-      "mean_token_accuracy": 0.513639942780137,
-      "num_input_tokens_seen": 356956832,
+      "epoch": 0.1689877333011452,
+      "grad_norm": 0.8828125,
+      "learning_rate": 1.0561255280627642e-05,
+      "loss": 2.611,
+      "mean_token_accuracy": 0.5080171688646078,
+      "num_input_tokens_seen": 337553504,
       "num_tokens": 142378323.0,
       "step": 700
     },
     {
-      "epoch": 0.18105555387911523,
-      "grad_norm": 0.486328125,
-      "learning_rate": 2.2631261315630658e-05,
-      "loss": 2.5395,
-      "mean_token_accuracy": 0.5144028599560261,
-      "num_input_tokens_seen": 382529376,
+      "epoch": 0.18105828567979843,
+      "grad_norm": 0.76171875,
+      "learning_rate": 1.1315630657815329e-05,
+      "loss": 2.6039,
+      "mean_token_accuracy": 0.5081416912004352,
+      "num_input_tokens_seen": 361961104,
       "num_tokens": 152623392.0,
       "step": 750
     },
     {
-      "epoch": 0.19312592413772292,
-      "grad_norm": 0.5234375,
-      "learning_rate": 2.4140012070006035e-05,
-      "loss": 2.5309,
-      "mean_token_accuracy": 0.5156180351227522,
-      "num_input_tokens_seen": 408021056,
+      "epoch": 0.19312883805845166,
+      "grad_norm": 0.7421875,
+      "learning_rate": 1.2070006035003018e-05,
+      "loss": 2.5936,
+      "mean_token_accuracy": 0.5093595118448139,
+      "num_input_tokens_seen": 386105904,
       "num_tokens": 162827766.0,
       "step": 800
     },
     {
-      "epoch": 0.2051962943963306,
-      "grad_norm": 0.447265625,
-      "learning_rate": 2.5648762824381413e-05,
-      "loss": 2.5241,
-      "mean_token_accuracy": 0.5165234027802944,
-      "num_input_tokens_seen": 433622464,
+      "epoch": 0.2051993904371049,
+      "grad_norm": 0.86328125,
+      "learning_rate": 1.2824381412190706e-05,
+      "loss": 2.5887,
+      "mean_token_accuracy": 0.5100815147906542,
+      "num_input_tokens_seen": 410248672,
       "num_tokens": 172979899.0,
       "step": 850
     },
     {
-      "epoch": 0.2172666646549383,
-      "grad_norm": 0.55078125,
-      "learning_rate": 2.7157513578756787e-05,
-      "loss": 2.5219,
-      "mean_token_accuracy": 0.5160751658678054,
-      "num_input_tokens_seen": 459074080,
+      "epoch": 0.2172699428157581,
+      "grad_norm": 0.81640625,
+      "learning_rate": 1.3578756789378394e-05,
+      "loss": 2.5903,
+      "mean_token_accuracy": 0.5090210309624672,
+      "num_input_tokens_seen": 434357040,
       "num_tokens": 183138615.0,
       "step": 900
     },
     {
-      "epoch": 0.22933703491354598,
-      "grad_norm": 0.482421875,
-      "learning_rate": 2.866626433313217e-05,
-      "loss": 2.511,
-      "mean_token_accuracy": 0.5176014001667499,
-      "num_input_tokens_seen": 484598624,
+      "epoch": 0.22934049519441133,
+      "grad_norm": 0.69921875,
+      "learning_rate": 1.4333132166566086e-05,
+      "loss": 2.5816,
+      "mean_token_accuracy": 0.5105240726843476,
+      "num_input_tokens_seen": 458609136,
       "num_tokens": 193253896.0,
       "step": 950
     },
     {
-      "epoch": 0.24140740517215364,
-      "grad_norm": 0.6015625,
-      "learning_rate": 3.0175015087507546e-05,
-      "loss": 2.4857,
-      "num_input_tokens_seen": 510077440,
+      "epoch": 0.24141104757306456,
+      "grad_norm": 0.64453125,
+      "learning_rate": 1.5087507543753773e-05,
+      "loss": 2.558,
+      "num_input_tokens_seen": 482701824,
       "step": 1000
     },
     {
-      "epoch": 0.24140740517215364,
-      "eval_loss": 2.4022138118743896,
-      "eval_mean_token_accuracy": 0.5380484161312853,
+      "epoch": 0.24141104757306456,
+      "eval_loss": 2.4748051166534424,
+      "eval_mean_token_accuracy": 0.529798322766908,
       "eval_num_tokens": 203431197.0,
-      "eval_runtime": 126.7655,
-      "eval_samples_per_second": 84.503,
-      "eval_steps_per_second": 21.126,
-      "num_input_tokens_seen": 510077440,
+      "eval_runtime": 125.5514,
+      "eval_samples_per_second": 85.32,
+      "eval_steps_per_second": 21.33,
+      "num_input_tokens_seen": 482701824,
       "step": 1000
     }
   ],
   "logging_steps": 50,
-  "max_steps": 8284,
-  "num_input_tokens_seen": 510077440,
-  "num_train_epochs": 2,
+  "max_steps": 16568,
+  "num_input_tokens_seen": 482701824,
+  "num_train_epochs": 4,
   "save_steps": 1000,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -245,8 +245,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.364506935558144e+17,
-  "train_batch_size": 32,
+  "total_flos": 1.2912744908980224e+17,
+  "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:02b13fc8d7c893a956ab01c588fd523c765a348a70de0153e06e4429d50ed8ee
+oid sha256:f02bab4e330ba535e68453b550f936b9689badf0448db28cf610480cc253d3fb
 size 5752