Training in progress, step 800

checkpoint-400/latest

@@ -1 +0,0 @@
-global_step400

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad8eaec1de240ce4628aadb21bff3950a01dd7ed74435382147288d9c92ceb61
+oid sha256:0233dc4a8194f3bef960787af2ec91f0e31a083f8d374af882f6450af5e55451
 size 10107626487

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c46199fffea0e99425fd5b72fb9d9f38616e7f3e5e72a777144d863f5d1c074b
+oid sha256:381c023978239b23cc247f99fe963167a8628446b44e933551317dfe31264784
 size 10107626487

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d1ccdebec9dc875e94daf07b5a6bd01f43fd477d4a04b82f3faa559bb5f8c92
+oid sha256:768eeaca4fb869b895451ff1734b628eb698e54a8670ec4887523af22d518950
 size 10107626487

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:034fb9fa2ab2e84404fc5d47168d481f029ff97ac31b3a7bdd5e20c2e41c3e65
+oid sha256:d0935cff42c9223a77c2953c7ff69a5cccd7b8230277cec7f703f5025371df9f
 size 10107626487

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8097fccfd55e74e350747af8b0b1ee043325954ece5f9ff102c60638793523ab
+oid sha256:d119d36fcfb5bdf735763ac754c7164bf5861667ed232ff2ee1f521df9cf46ce
 size 10107626487

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f1278964e783bf13b21aeb26452cc546a34c7eabe1744718cc71ddbf680481a
+oid sha256:72ba38f38dc3469211188737f44ee539df6f0d2f1113e8c9808bd498a97ba56e
 size 10107626487

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f127d028306a0a579a34daa0602b0a128155150f80b92b85299cc8dd8d073287
+oid sha256:bc9ad0aa2d656102bbfe892890bc18b332901104947c2e8a0cc11e68d656bb54
 size 10107626487

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:179b3c4a09d744e1fdb711b0dce7203df5c336388bd692b2ef05bbc69badf222
+oid sha256:2ac9f429f41b6690c59415bdc4643bc040c204301bf6d9dc7f032d741bf2975f
 size 10107626487

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:05bc3d711fff7b3c9125f3c44871eb24f03502e90e2912080183b8d173d6d2bf
+oid sha256:233d3f4e42ae90fa4994c8c0b577e9fb31018d32bc58244df6cd4e57a5bc314a
 size 168086

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ee59ff02e2704ed398fe32cb78313e36c2189e2f7d3d3169e2b40abbb698d14
+oid sha256:f8e6e08c9fbe96968cef4593a608fa7f32d2a76db208b81ed1cd4a8b64b512d6
 size 168086

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc398ec3f74fb18dcd6f44326db86ee1f71c18432ce93b82d0df95ca9ea59408
+oid sha256:70f0a38b369c454c21cacfd4775ffd52d06a950af9007bb13eb599fb154fbd16
 size 168086

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a5f3976f71e61f3655290a33f4fb1cce58f4d992a77dff9a8b4d0d112d754e61
+oid sha256:220292ca12becbd9ff5ee9e1cf8127dad0790ad867f8a9dc4b70a41a9328df3a
 size 168086

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d10e4c5407c6ea57e678793a825a4fe44bb9482ffef589fa22b56f622195d77b
+oid sha256:5522cef3c853ba1a9d7d551d07427d3538f3363e964bd8c9926b829a7452f4f1
 size 168086

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a8133a4e33bb8f7cff8537525033dacab44d67bd130ee3a4a237ca6257f4f1e
+oid sha256:4448f725283b27cf989c1c0dfd6702b9f83e6c23f68ce76d3e4ddf4b8760f6fa
 size 168086

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1fc82e0e77154feaf019a2714b9d1183ab77a9975cee276f89216b93a4b6578e
+oid sha256:84704b403c899d7143fd8224463da6540f81844bef520713bfccadc09e163505
 size 168086

{checkpoint-400/global_step400 → checkpoint-800/global_step800}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f4649f3451237369e5d7008539026ff33d08263e2b9958118ca906955d4c0ae7
+oid sha256:713fcf20cfef9dfef3231bc5a8c499e8e943d47163a4393d8f733c8c8b664ad7
 size 168086

checkpoint-800/latest

@@ -0,0 +1 @@
+global_step800

{checkpoint-400 → checkpoint-800}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cba47968844811d5694cb13c342091f18cc17562005847525dc4c22079e66d59
+oid sha256:072428d75e810d41d1cc6413f6e08761a39f896b97abd7cd47a986b9d58c1823
 size 9976576392

{checkpoint-400 → checkpoint-800}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:98be5496bae8e5e948b3fe4c8ed95aa0f1f11056a9a771f30fa5b177beb379de
+oid sha256:33524383bb67b7cd8c24780afd4c26119fa1b45d6e6c9d7fa921ff3a85b3a78d
 size 3500296504

{checkpoint-400 → checkpoint-800}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.13333333333333333,
-  "global_step": 400,
+  "epoch": 0.26666666666666666,
+  "global_step": 800,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -246,11 +246,251 @@
       "learning_rate": 0.0003,
       "loss": 4.4799,
       "step": 400
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0003,
+      "loss": 4.3733,
+      "step": 410
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0003,
+      "loss": 4.1788,
+      "step": 420
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0003,
+      "loss": 3.6706,
+      "step": 430
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0003,
+      "loss": 2.8767,
+      "step": 440
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0003,
+      "loss": 2.3927,
+      "step": 450
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0003,
+      "loss": 2.3062,
+      "step": 460
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0003,
+      "loss": 2.2299,
+      "step": 470
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0003,
+      "loss": 2.1739,
+      "step": 480
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0003,
+      "loss": 2.1303,
+      "step": 490
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0003,
+      "loss": 2.1092,
+      "step": 500
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0003,
+      "loss": 2.0715,
+      "step": 510
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0003,
+      "loss": 2.0473,
+      "step": 520
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0003,
+      "loss": 2.0566,
+      "step": 530
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0003,
+      "loss": 1.9818,
+      "step": 540
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0003,
+      "loss": 1.9592,
+      "step": 550
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0003,
+      "loss": 1.8939,
+      "step": 560
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0003,
+      "loss": 1.8441,
+      "step": 570
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0003,
+      "loss": 1.808,
+      "step": 580
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0003,
+      "loss": 1.765,
+      "step": 590
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0003,
+      "loss": 1.706,
+      "step": 600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0003,
+      "loss": 1.6328,
+      "step": 610
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003,
+      "loss": 1.6197,
+      "step": 620
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003,
+      "loss": 1.6106,
+      "step": 630
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003,
+      "loss": 1.5649,
+      "step": 640
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003,
+      "loss": 1.5001,
+      "step": 650
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003,
+      "loss": 1.4926,
+      "step": 660
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003,
+      "loss": 1.4682,
+      "step": 670
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003,
+      "loss": 1.442,
+      "step": 680
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003,
+      "loss": 1.4117,
+      "step": 690
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003,
+      "loss": 1.379,
+      "step": 700
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003,
+      "loss": 1.3609,
+      "step": 710
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003,
+      "loss": 1.3224,
+      "step": 720
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003,
+      "loss": 1.2853,
+      "step": 730
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003,
+      "loss": 1.2762,
+      "step": 740
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003,
+      "loss": 1.2265,
+      "step": 750
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003,
+      "loss": 1.2232,
+      "step": 760
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003,
+      "loss": 1.2075,
+      "step": 770
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003,
+      "loss": 1.168,
+      "step": 780
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003,
+      "loss": 1.1472,
+      "step": 790
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003,
+      "loss": 1.118,
+      "step": 800
     }
   ],
   "max_steps": 3000,
   "num_train_epochs": 9223372036854775807,
-  "total_flos": 167503724544000.0,
+  "total_flos": 335007449088000.0,
   "trial_name": null,
   "trial_params": null
 }

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ebc5354d2855ef417ffbf8fe045db7b97365794e8867bcd1a217a43e697f39f
-size 15141
+oid sha256:b518fc726384856248944d1be48f0582a8e8298752371b3b3f5437577456a8b7
+size 16711