Training in progress, step 1000

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:66eb01a0e90463b9da96b92c71aa113fb0cc395d468d494cbaaf30b0c996c900
+oid sha256:5703ed353a444bcb887496e437dcc7b16f8bdaae831e4e6d4f7ae38a220a5df3
 size 10107626487

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a9c27857331e866aa180ca994d30ecdeb152ce200686244186589fd2b6c5399
+oid sha256:65d6937e778c81679ecadc56c59bb90e942cd63ffc72959111931f27b22bea57
 size 10107626487

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:94cc70fd746640a3708551bd27076cca0c4493fc0ec627adc422dc06e217b4f1
+oid sha256:7078c4a9cb10247701dadff610cea1a671df2436f25c85ce7ec7f830d5864cff
 size 10107626487

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:53f785e4989207b0d802f03d21811f0310e6887f491becf18aab59dc2a8e34dc
+oid sha256:632c044c424e36e9ac4d04c3db4d08e0453493e42ad88a6bbfdceadec83fd685
 size 10107626487

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:055b189cd86f4ab7885ffd7733ca11939b1c02c7b961887c23ff3765b85b2c26
+oid sha256:c31e28791a65e07de46a9f174419774d315db15b86b1c34705af7a4bc7d99a66
 size 10107626487

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b598e84707f1b589d6e34c06fddfcfd1c635efdeb8c34cf5ca13a510f0949837
+oid sha256:07b03db6da0fdf58a5abbc86c3e32353bc4ea225df4814dd9b0c87094cba0a38
 size 10107626487

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15519b6df3b12db386d8c11e652af04badf693c055d7c3d49091aad19745e337
+oid sha256:1621abeb507c8e04e54646297d7dfc104a12de8750d5ef2e450703166e2e6a23
 size 10107626487

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a17fc5b79f53abceedb3cc4c49fba56335c0c00a6ffa5320a11cacac9ad4e106
+oid sha256:eb1c8f3f16701d2f9fe90ecb59b247f72b3a0bde6d3db2267a5c44807cf77842
 size 10107626487

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c17dd6ac6428c5d94c99148aa50c12df93df8b2462cbf19e8e84d9d333fcbd1e
+oid sha256:4658b6b3c7f7a48dd296534d1ec3dc6d546541ef01a0a3c11c403c23380b9050
 size 168086

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6dade817edf76921be5c0796cbcfdf915c904c183acf9c931777f623390eb1f3
+oid sha256:fc1c982bfd73913563caf1b20156081f202141ef2b4942de7b04a60a2735599a
 size 168086

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:87c1c5de42abbdea85c5b809d6ad1ea6ea81c0292569eae30054d35b7568f9e0
+oid sha256:3c42c53335aafb877cfbba5749485ee3cd61082a493faf56f57f8709eb5e6252
 size 168086

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d2bc5f999fb8ccc85ce968f9308e1832cc481d0a36fbfee21231ee0251223b9
+oid sha256:ccd7e1a57d35ed384ee88e36e9a684ec710295b14dacde4ea9e97a2e0dd5e2f4
 size 168086

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:287d7ae815a009508c9d5763cdb2b95cdccbcb41047461ddb5942fe4be828cd4
+oid sha256:aa0a5cc62566f10c81232feb782efb1f9f3ef3a38928981584b06095d904ad93
 size 168086

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c563d457d97887b6b8ebca491ddbbb5f7d02940671baa85e856e70f2373dd1e8
+oid sha256:8585a4986217ec7db11028c99ffd2fb8f753ad9973841c9b2be8b872fa0f5af7
 size 168086

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bbaa5dfcd7813a949984070a065f4d96fd979a8486c397c0fe82e466df55719e
+oid sha256:689aff8732618feb10643e53dc24544b9e68c3fe79059dd39c4324f792c95f8e
 size 168086

{checkpoint-600/global_step600 → checkpoint-1000/global_step1000}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2cfe0ea806988e2741a7bc673de2fae1d546b08c34f0cf088d122f77c3098bac
+oid sha256:200527511d1203250da233f27a50a276bf203ca991380ba5c7c096f1955a2752
 size 168086

checkpoint-1000/latest

@@ -0,0 +1 @@
+global_step1000

{checkpoint-600 → checkpoint-1000}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d5217aa0cdf2a3e998ac2276247c8b950f52424ef53ba6bedbb47fe566772c4
+oid sha256:11913c7f1753ca039f36c4c2b6bbefb982404ad4203fe8ba96a2f892793f3922
 size 9976576392

{checkpoint-600 → checkpoint-1000}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15c35bdf2b64df17b13e8aa89bf76a22e3fe647fb7a427d6d581f7b18dbbea71
+oid sha256:b9877b1573d48279b0f52a3d16935fdb3bf67e40baa64bc8774bbb8005dd3519
 size 3500296504

{checkpoint-600 → checkpoint-1000}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.2,
-  "global_step": 600,
+  "epoch": 0.3333333333333333,
+  "global_step": 1000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -366,11 +366,251 @@
       "learning_rate": 0.0003,
       "loss": 1.706,
       "step": 600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0003,
+      "loss": 1.6328,
+      "step": 610
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003,
+      "loss": 1.6197,
+      "step": 620
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003,
+      "loss": 1.6106,
+      "step": 630
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003,
+      "loss": 1.5649,
+      "step": 640
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003,
+      "loss": 1.5001,
+      "step": 650
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003,
+      "loss": 1.4926,
+      "step": 660
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003,
+      "loss": 1.4682,
+      "step": 670
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003,
+      "loss": 1.442,
+      "step": 680
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003,
+      "loss": 1.4117,
+      "step": 690
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003,
+      "loss": 1.379,
+      "step": 700
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003,
+      "loss": 1.3609,
+      "step": 710
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003,
+      "loss": 1.3224,
+      "step": 720
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003,
+      "loss": 1.2853,
+      "step": 730
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003,
+      "loss": 1.2762,
+      "step": 740
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003,
+      "loss": 1.2265,
+      "step": 750
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003,
+      "loss": 1.2232,
+      "step": 760
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003,
+      "loss": 1.2075,
+      "step": 770
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003,
+      "loss": 1.168,
+      "step": 780
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003,
+      "loss": 1.1472,
+      "step": 790
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003,
+      "loss": 1.118,
+      "step": 800
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003,
+      "loss": 1.1228,
+      "step": 810
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003,
+      "loss": 1.1339,
+      "step": 820
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003,
+      "loss": 1.0853,
+      "step": 830
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003,
+      "loss": 1.0676,
+      "step": 840
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003,
+      "loss": 1.0905,
+      "step": 850
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003,
+      "loss": 1.076,
+      "step": 860
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003,
+      "loss": 1.0202,
+      "step": 870
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003,
+      "loss": 1.0123,
+      "step": 880
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003,
+      "loss": 0.9863,
+      "step": 890
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003,
+      "loss": 0.9347,
+      "step": 900
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003,
+      "loss": 0.9416,
+      "step": 910
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003,
+      "loss": 0.9165,
+      "step": 920
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003,
+      "loss": 0.8996,
+      "step": 930
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003,
+      "loss": 0.8673,
+      "step": 940
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003,
+      "loss": 0.8449,
+      "step": 950
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003,
+      "loss": 0.8468,
+      "step": 960
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003,
+      "loss": 0.817,
+      "step": 970
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003,
+      "loss": 0.7947,
+      "step": 980
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003,
+      "loss": 0.7706,
+      "step": 990
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003,
+      "loss": 0.7357,
+      "step": 1000
     }
   ],
   "max_steps": 3000,
   "num_train_epochs": 9223372036854775807,
-  "total_flos": 251255586816000.0,
+  "total_flos": 418759311360000.0,
   "trial_name": null,
   "trial_params": null
 }

checkpoint-600/latest

@@ -1 +0,0 @@
-global_step600

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf0edb971b8a90918c57c5b33c853a049ca331754786d7d176c54919449c9aaa
-size 18281
+oid sha256:1fef6847951e7a7a5d7472c77aeb6d8f614b223d630dfc7f7950ed07e82dfab5
+size 19851