Training in progress, step 1200

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0233dc4a8194f3bef960787af2ec91f0e31a083f8d374af882f6450af5e55451
+oid sha256:4fb262812656b3c83b6881b1dd67c5767d8042011235ae6468454646240b5c07
 size 10107626487

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:381c023978239b23cc247f99fe963167a8628446b44e933551317dfe31264784
+oid sha256:7df89141b87c7b3401cf3dc6d0f0e35b07ee4007b03d264482fa67df22e6e386
 size 10107626487

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:768eeaca4fb869b895451ff1734b628eb698e54a8670ec4887523af22d518950
+oid sha256:304104b8210689a208414ce338a1a8170d07fd50ede690c06cc17f711af7d23b
 size 10107626487

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d0935cff42c9223a77c2953c7ff69a5cccd7b8230277cec7f703f5025371df9f
+oid sha256:11e7a70ad03df41c4584486b35cde5c60d0c66a22c3f5c4048017d438d1d7dd5
 size 10107626487

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d119d36fcfb5bdf735763ac754c7164bf5861667ed232ff2ee1f521df9cf46ce
+oid sha256:11e5c69f087a06dfa4218e7205637830d59e2eb6ab4496e20b6099fcccbc9b07
 size 10107626487

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72ba38f38dc3469211188737f44ee539df6f0d2f1113e8c9808bd498a97ba56e
+oid sha256:37dd48878dd48bb7d2a1f2ea052444d84a13e6f4cbab8f1d27904757ae116bd0
 size 10107626487

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc9ad0aa2d656102bbfe892890bc18b332901104947c2e8a0cc11e68d656bb54
+oid sha256:4d614936212979e10ebba5d41469df3f0f5d966c2da7d3029cfa1ec1e9dd897a
 size 10107626487

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ac9f429f41b6690c59415bdc4643bc040c204301bf6d9dc7f032d741bf2975f
+oid sha256:3266f649233e5304749c7c1f07769559935f00b03922e28bff0442991ec9388a
 size 10107626487

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:233d3f4e42ae90fa4994c8c0b577e9fb31018d32bc58244df6cd4e57a5bc314a
+oid sha256:ee838bbe2d8ed873c80d9a5740fcb7f54c91e439e0f4bdde882b4456ea232c7c
 size 168086

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8e6e08c9fbe96968cef4593a608fa7f32d2a76db208b81ed1cd4a8b64b512d6
+oid sha256:4326e6a49ee2354fb513f4f301a33c21ffb84a9a51a4e3d0ed33bc12d02bd10b
 size 168086

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70f0a38b369c454c21cacfd4775ffd52d06a950af9007bb13eb599fb154fbd16
+oid sha256:d6c42d8a6eeae2786121f541dc5d4a06728de74fdd785c4e82bf7be14708e787
 size 168086

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:220292ca12becbd9ff5ee9e1cf8127dad0790ad867f8a9dc4b70a41a9328df3a
+oid sha256:ca8f493f0fa1429963ac77a6e815a9f189220490b4f4457f7be2b942cdc01726
 size 168086

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5522cef3c853ba1a9d7d551d07427d3538f3363e964bd8c9926b829a7452f4f1
+oid sha256:7c1911bcfb001e09fe12e3ee69beaea09f5f021dc647d8b1040b0d66821cc448
 size 168086

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4448f725283b27cf989c1c0dfd6702b9f83e6c23f68ce76d3e4ddf4b8760f6fa
+oid sha256:6c7c6cda3f5f3d82146b47b1ad4b836ecfff16ca2e586ea073dd665626f8757e
 size 168086

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84704b403c899d7143fd8224463da6540f81844bef520713bfccadc09e163505
+oid sha256:23cb97ab9d849ada9216fd67b47e13c5ebfbeaeb019c59c09a9cb3298aa097e6
 size 168086

{checkpoint-800/global_step800 → checkpoint-1200/global_step1200}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:713fcf20cfef9dfef3231bc5a8c499e8e943d47163a4393d8f733c8c8b664ad7
+oid sha256:5bb17570ce5f68b67d4232815d8eb3b18765ca6071753b7ef99040ac8710b3ec
 size 168086

checkpoint-1200/latest

@@ -0,0 +1 @@
+global_step1200

{checkpoint-800 → checkpoint-1200}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:072428d75e810d41d1cc6413f6e08761a39f896b97abd7cd47a986b9d58c1823
+oid sha256:050176f4baa1bd8c33427c5122e69e830566029acf7a4cca7415b3b41047fcbd
 size 9976576392

{checkpoint-800 → checkpoint-1200}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:33524383bb67b7cd8c24780afd4c26119fa1b45d6e6c9d7fa921ff3a85b3a78d
+oid sha256:30aa4a97478659826d8dbb57cb3e6ba545204647306edfcfec56c36a5b191c85
 size 3500296504

{checkpoint-800 → checkpoint-1200}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.26666666666666666,
-  "global_step": 800,
+  "epoch": 0.4,
+  "global_step": 1200,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -486,11 +486,251 @@
       "learning_rate": 0.0003,
       "loss": 1.118,
       "step": 800
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003,
+      "loss": 1.1228,
+      "step": 810
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003,
+      "loss": 1.1339,
+      "step": 820
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003,
+      "loss": 1.0853,
+      "step": 830
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003,
+      "loss": 1.0676,
+      "step": 840
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003,
+      "loss": 1.0905,
+      "step": 850
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003,
+      "loss": 1.076,
+      "step": 860
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003,
+      "loss": 1.0202,
+      "step": 870
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003,
+      "loss": 1.0123,
+      "step": 880
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003,
+      "loss": 0.9863,
+      "step": 890
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003,
+      "loss": 0.9347,
+      "step": 900
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003,
+      "loss": 0.9416,
+      "step": 910
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003,
+      "loss": 0.9165,
+      "step": 920
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003,
+      "loss": 0.8996,
+      "step": 930
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003,
+      "loss": 0.8673,
+      "step": 940
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003,
+      "loss": 0.8449,
+      "step": 950
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003,
+      "loss": 0.8468,
+      "step": 960
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003,
+      "loss": 0.817,
+      "step": 970
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003,
+      "loss": 0.7947,
+      "step": 980
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003,
+      "loss": 0.7706,
+      "step": 990
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003,
+      "loss": 0.7357,
+      "step": 1000
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003,
+      "loss": 0.6983,
+      "step": 1010
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003,
+      "loss": 0.6989,
+      "step": 1020
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003,
+      "loss": 0.6692,
+      "step": 1030
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003,
+      "loss": 0.6553,
+      "step": 1040
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003,
+      "loss": 0.6272,
+      "step": 1050
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003,
+      "loss": 0.6092,
+      "step": 1060
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003,
+      "loss": 0.604,
+      "step": 1070
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003,
+      "loss": 0.6494,
+      "step": 1080
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003,
+      "loss": 0.6155,
+      "step": 1090
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003,
+      "loss": 0.5713,
+      "step": 1100
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003,
+      "loss": 0.5358,
+      "step": 1110
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003,
+      "loss": 0.5005,
+      "step": 1120
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003,
+      "loss": 0.483,
+      "step": 1130
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003,
+      "loss": 0.4736,
+      "step": 1140
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003,
+      "loss": 0.4585,
+      "step": 1150
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003,
+      "loss": 0.4442,
+      "step": 1160
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003,
+      "loss": 0.4392,
+      "step": 1170
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003,
+      "loss": 0.4084,
+      "step": 1180
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003,
+      "loss": 0.4104,
+      "step": 1190
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003,
+      "loss": 0.3855,
+      "step": 1200
     }
   ],
   "max_steps": 3000,
   "num_train_epochs": 9223372036854775807,
-  "total_flos": 335007449088000.0,
+  "total_flos": 502511173632000.0,
   "trial_name": null,
   "trial_params": null
 }

checkpoint-800/latest

@@ -1 +0,0 @@
-global_step800

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9195eb0ab07f76500bcac0b0eec3ca51997f05cae782877112fd61a0f25f510c
-size 21421
+oid sha256:80c6f49f860f2ea4ccc6f90490738df78763e614072a048d7839d11aebd11a94
+size 22991