Training in progress, step 141000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6e43382fe5ddb78fed06a23ba6c7b8489c50f8ee7949d8db86e49cd8910036e
+oid sha256:a27f87288d8d797a749da5bf4d352cdabd92413a2e35e052af216c7df1f69945
 size 1410301944

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c18874d88aac76ea7c7006e997509fca95df88b10d2c13b5a6816de7643ed6e
+oid sha256:36756e75b3466f2e619ffcb01fde732bcbed6a8bb6e17f933bd8b701f263e4f2
 size 2820185786

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:82993dca9aea22266a253201514efb5478f36bf5a374573dc48fbab5e03c52d6
+oid sha256:95aeb3e8ddbb19f44b8ac55566129494d59b1f0669d87d7f6b45254087f1767e
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf74877c1fcc66d6df58cb7c2b28db5c3be81aec77034ec2a9ace3e30449eb22
+oid sha256:e7f2c2062cd4eab2105e1d3af30621ba0055a18128fac0ce700a512d64dfcfc4
 size 1064

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.335609430339745,
+  "epoch": 3.097265706246529,
   "eval_steps": 500,
-  "global_step": 140000,
+  "global_step": 141000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -24928,12 +24928,190 @@
       "eval_steps_per_second": 15.131,
       "num_input_tokens_seen": 73388446624,
       "step": 140000
+    },
+    {
+      "epoch": 3.076397950841334,
+      "grad_norm": 0.09252593666315079,
+      "learning_rate": 0.0001,
+      "loss": 2.3535,
+      "num_input_tokens_seen": 73440875424,
+      "step": 140050
+    },
+    {
+      "epoch": 3.0774962537573973,
+      "grad_norm": 0.08520153909921646,
+      "learning_rate": 0.0001,
+      "loss": 2.3529,
+      "num_input_tokens_seen": 73493304224,
+      "step": 140100
+    },
+    {
+      "epoch": 3.07859455667346,
+      "grad_norm": 0.09475487470626831,
+      "learning_rate": 0.0001,
+      "loss": 2.3539,
+      "num_input_tokens_seen": 73545729952,
+      "step": 140150
+    },
+    {
+      "epoch": 3.079692859589523,
+      "grad_norm": 0.08525670319795609,
+      "learning_rate": 0.0001,
+      "loss": 2.3603,
+      "num_input_tokens_seen": 73598155232,
+      "step": 140200
+    },
+    {
+      "epoch": 3.080791162505586,
+      "grad_norm": 0.09414695203304291,
+      "learning_rate": 0.0001,
+      "loss": 2.3596,
+      "num_input_tokens_seen": 73650584032,
+      "step": 140250
+    },
+    {
+      "epoch": 3.0818894654216487,
+      "grad_norm": 0.08829599618911743,
+      "learning_rate": 0.0001,
+      "loss": 2.3582,
+      "num_input_tokens_seen": 73703009408,
+      "step": 140300
+    },
+    {
+      "epoch": 3.082987768337712,
+      "grad_norm": 0.08346480131149292,
+      "learning_rate": 0.0001,
+      "loss": 2.3473,
+      "num_input_tokens_seen": 73755435104,
+      "step": 140350
+    },
+    {
+      "epoch": 3.0840860712537745,
+      "grad_norm": 0.09302923828363419,
+      "learning_rate": 0.0001,
+      "loss": 2.3555,
+      "num_input_tokens_seen": 73807860000,
+      "step": 140400
+    },
+    {
+      "epoch": 3.0851843741698373,
+      "grad_norm": 0.08695721626281738,
+      "learning_rate": 0.0001,
+      "loss": 2.3578,
+      "num_input_tokens_seen": 73860288800,
+      "step": 140450
+    },
+    {
+      "epoch": 3.0862826770859004,
+      "grad_norm": 0.09424284100532532,
+      "learning_rate": 0.0001,
+      "loss": 2.3523,
+      "num_input_tokens_seen": 73912717600,
+      "step": 140500
+    },
+    {
+      "epoch": 3.0862826770859004,
+      "eval_loss": 2.2698493003845215,
+      "eval_runtime": 81.2331,
+      "eval_samples_per_second": 61.551,
+      "eval_steps_per_second": 15.388,
+      "num_input_tokens_seen": 73912717600,
+      "step": 140500
+    },
+    {
+      "epoch": 3.087380980001963,
+      "grad_norm": 0.08606674522161484,
+      "learning_rate": 0.0001,
+      "loss": 2.3589,
+      "num_input_tokens_seen": 73965145984,
+      "step": 140550
+    },
+    {
+      "epoch": 3.0884792829180263,
+      "grad_norm": 0.09220123291015625,
+      "learning_rate": 0.0001,
+      "loss": 2.3503,
+      "num_input_tokens_seen": 74017574784,
+      "step": 140600
+    },
+    {
+      "epoch": 3.089577585834089,
+      "grad_norm": 0.10021138191223145,
+      "learning_rate": 0.0001,
+      "loss": 2.3528,
+      "num_input_tokens_seen": 74070003040,
+      "step": 140650
+    },
+    {
+      "epoch": 3.0906758887501518,
+      "grad_norm": 0.08400563895702362,
+      "learning_rate": 0.0001,
+      "loss": 2.3575,
+      "num_input_tokens_seen": 74122431840,
+      "step": 140700
+    },
+    {
+      "epoch": 3.091774191666215,
+      "grad_norm": 0.08861430734395981,
+      "learning_rate": 0.0001,
+      "loss": 2.3552,
+      "num_input_tokens_seen": 74174859680,
+      "step": 140750
+    },
+    {
+      "epoch": 3.0928724945822776,
+      "grad_norm": 0.08466708660125732,
+      "learning_rate": 0.0001,
+      "loss": 2.3603,
+      "num_input_tokens_seen": 74227284768,
+      "step": 140800
+    },
+    {
+      "epoch": 3.0939707974983404,
+      "grad_norm": 0.08707701414823532,
+      "learning_rate": 0.0001,
+      "loss": 2.3595,
+      "num_input_tokens_seen": 74279711840,
+      "step": 140850
+    },
+    {
+      "epoch": 3.0950691004144035,
+      "grad_norm": 0.08657340705394745,
+      "learning_rate": 0.0001,
+      "loss": 2.3511,
+      "num_input_tokens_seen": 74332140640,
+      "step": 140900
+    },
+    {
+      "epoch": 3.0961674033304663,
+      "grad_norm": 0.08521311730146408,
+      "learning_rate": 0.0001,
+      "loss": 2.3569,
+      "num_input_tokens_seen": 74384569440,
+      "step": 140950
+    },
+    {
+      "epoch": 3.097265706246529,
+      "grad_norm": 0.08738870918750763,
+      "learning_rate": 0.0001,
+      "loss": 2.3587,
+      "num_input_tokens_seen": 74436998240,
+      "step": 141000
+    },
+    {
+      "epoch": 3.097265706246529,
+      "eval_loss": 2.269127607345581,
+      "eval_runtime": 80.825,
+      "eval_samples_per_second": 61.862,
+      "eval_steps_per_second": 15.466,
+      "num_input_tokens_seen": 74436998240,
+      "step": 141000
     }
   ],
   "logging_steps": 50,
-  "max_steps": 140000,
-  "num_input_tokens_seen": 73388446624,
-  "num_train_epochs": 2,
+  "max_steps": 200000,
+  "num_input_tokens_seen": 74436998240,
+  "num_train_epochs": 5,
   "save_steps": 1000,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -24942,12 +25120,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 1.2988416447181578e+20,
+  "total_flos": 1.3173990957632102e+20,
   "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d0658393df628109da44bf73cd2c5b7b1aaaea9c34af48a5725df7cb08bc1427
+oid sha256:98ea109117710c0c998ea268594e6a7d0e86331c406b4b50e21b67f4948ff266
 size 6008