Training in progress, step 600

checkpoint-200/latest

@@ -1 +0,0 @@
-global_step200

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:71d2f9ca99b2c8752d5acd8c1555a01f48f2d977f249dd1f70f224809fc435f6
+oid sha256:66eb01a0e90463b9da96b92c71aa113fb0cc395d468d494cbaaf30b0c996c900
 size 10107626487

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f47da0e5552c15f333a9fd0db761ede06034ac687cdd21456980f759ee21e8a8
+oid sha256:1a9c27857331e866aa180ca994d30ecdeb152ce200686244186589fd2b6c5399
 size 10107626487

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:043a4a45cef8110b8e1abf415169f560134e1070f36c115e8b793d56d92a9857
+oid sha256:94cc70fd746640a3708551bd27076cca0c4493fc0ec627adc422dc06e217b4f1
 size 10107626487

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d25278e6c3183c87ff05e24b5f73e31e348f5f49708ae5a6a2f379bed5d99274
+oid sha256:53f785e4989207b0d802f03d21811f0310e6887f491becf18aab59dc2a8e34dc
 size 10107626487

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f6246741e64c84748a0a5021600606f684a38ed68eccec43754832e49d496e5
+oid sha256:055b189cd86f4ab7885ffd7733ca11939b1c02c7b961887c23ff3765b85b2c26
 size 10107626487

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b647e38e89a8c78d6adf4116b9380a5818e3e5155a62f69820f9d50472b46b5a
+oid sha256:b598e84707f1b589d6e34c06fddfcfd1c635efdeb8c34cf5ca13a510f0949837
 size 10107626487

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9208254f9c3fd4a0d7bd799d8f304c16b364f1c33b54df10499e9572370dcb22
+oid sha256:15519b6df3b12db386d8c11e652af04badf693c055d7c3d49091aad19745e337
 size 10107626487

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eb61950d5fc52bbbb13efa4dabadc3e4bdb7891654f30a7ee148674c610dbead
+oid sha256:a17fc5b79f53abceedb3cc4c49fba56335c0c00a6ffa5320a11cacac9ad4e106
 size 10107626487

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c34dac8a5115294c99ffe0ff0d6c8030d397e9e76a4c3cae10c6f57b5951ec26
+oid sha256:c17dd6ac6428c5d94c99148aa50c12df93df8b2462cbf19e8e84d9d333fcbd1e
 size 168086

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:297f1b081b4b97fa75810d998d3c622a62cde737ec4895f3fb93596432090506
+oid sha256:6dade817edf76921be5c0796cbcfdf915c904c183acf9c931777f623390eb1f3
 size 168086

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a5e2693442a9df7415519e489afddd763bc7416f8d2159d1f5ec200516f2de5
+oid sha256:87c1c5de42abbdea85c5b809d6ad1ea6ea81c0292569eae30054d35b7568f9e0
 size 168086

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:234a6a04b487bb30540f96fad960db6fa486d6cb3e11dca84b683d3455822ed4
+oid sha256:6d2bc5f999fb8ccc85ce968f9308e1832cc481d0a36fbfee21231ee0251223b9
 size 168086

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70cedfd022c0dd54d94c1f74f3f724b37365b3f63080c6b277275b92d068d938
+oid sha256:287d7ae815a009508c9d5763cdb2b95cdccbcb41047461ddb5942fe4be828cd4
 size 168086

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:25fa10cabd6744c9bb94903703359f4c94668a01727d8d601f3c9042de5edc34
+oid sha256:c563d457d97887b6b8ebca491ddbbb5f7d02940671baa85e856e70f2373dd1e8
 size 168086

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba591f0a7ee08808a1d41a17aa1fe43bfd502af61513671a76b2f3c84760bf8d
+oid sha256:bbaa5dfcd7813a949984070a065f4d96fd979a8486c397c0fe82e466df55719e
 size 168086

{checkpoint-200/global_step200 → checkpoint-600/global_step600}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:acff605da5d216a99e47f8ca6229a4fbdf7c25932dd3b8b0ba4d45bfc72ccbaf
+oid sha256:2cfe0ea806988e2741a7bc673de2fae1d546b08c34f0cf088d122f77c3098bac
 size 168086

checkpoint-600/latest

@@ -0,0 +1 @@
+global_step600

{checkpoint-200 → checkpoint-600}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:06b4a7088da3def00aeba160f50fbccff94c4b493b84b0b2d4ca97dba29c300f
+oid sha256:2d5217aa0cdf2a3e998ac2276247c8b950f52424ef53ba6bedbb47fe566772c4
 size 9976576392

{checkpoint-200 → checkpoint-600}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8a1f3f35590cedc71b241189f82c07f0f7391e458939e0fbe5763893ed0d349
+oid sha256:15c35bdf2b64df17b13e8aa89bf76a22e3fe647fb7a427d6d581f7b18dbbea71
 size 3500296504

{checkpoint-200 → checkpoint-600}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.06666666666666667,
-  "global_step": 200,
+  "epoch": 0.2,
+  "global_step": 600,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -126,11 +126,251 @@
       "learning_rate": 0.0003,
       "loss": 4.9139,
       "step": 200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0003,
+      "loss": 4.8974,
+      "step": 210
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0003,
+      "loss": 4.9036,
+      "step": 220
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0003,
+      "loss": 4.8635,
+      "step": 230
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0003,
+      "loss": 4.7883,
+      "step": 240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0003,
+      "loss": 4.7681,
+      "step": 250
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0003,
+      "loss": 4.7572,
+      "step": 260
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0003,
+      "loss": 4.7552,
+      "step": 270
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0003,
+      "loss": 4.706,
+      "step": 280
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0003,
+      "loss": 4.7015,
+      "step": 290
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0003,
+      "loss": 4.695,
+      "step": 300
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0003,
+      "loss": 4.6808,
+      "step": 310
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0003,
+      "loss": 4.6423,
+      "step": 320
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0003,
+      "loss": 4.613,
+      "step": 330
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0003,
+      "loss": 4.5851,
+      "step": 340
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0003,
+      "loss": 4.5882,
+      "step": 350
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0003,
+      "loss": 4.6228,
+      "step": 360
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0003,
+      "loss": 4.6269,
+      "step": 370
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0003,
+      "loss": 4.5364,
+      "step": 380
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0003,
+      "loss": 4.4992,
+      "step": 390
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0003,
+      "loss": 4.4799,
+      "step": 400
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0003,
+      "loss": 4.3733,
+      "step": 410
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0003,
+      "loss": 4.1788,
+      "step": 420
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0003,
+      "loss": 3.6706,
+      "step": 430
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0003,
+      "loss": 2.8767,
+      "step": 440
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0003,
+      "loss": 2.3927,
+      "step": 450
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0003,
+      "loss": 2.3062,
+      "step": 460
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0003,
+      "loss": 2.2299,
+      "step": 470
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0003,
+      "loss": 2.1739,
+      "step": 480
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0003,
+      "loss": 2.1303,
+      "step": 490
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0003,
+      "loss": 2.1092,
+      "step": 500
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0003,
+      "loss": 2.0715,
+      "step": 510
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0003,
+      "loss": 2.0473,
+      "step": 520
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0003,
+      "loss": 2.0566,
+      "step": 530
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0003,
+      "loss": 1.9818,
+      "step": 540
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0003,
+      "loss": 1.9592,
+      "step": 550
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0003,
+      "loss": 1.8939,
+      "step": 560
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0003,
+      "loss": 1.8441,
+      "step": 570
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0003,
+      "loss": 1.808,
+      "step": 580
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0003,
+      "loss": 1.765,
+      "step": 590
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0003,
+      "loss": 1.706,
+      "step": 600
     }
   ],
   "max_steps": 3000,
   "num_train_epochs": 9223372036854775807,
-  "total_flos": 83751862272000.0,
+  "total_flos": 251255586816000.0,
   "trial_name": null,
   "trial_params": null
 }

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7125e059489f5648613066e3ee25f5b3db395f89a0c3ea565999ab8326362e4e
-size 12001
+oid sha256:accaaa15e2b69671590fc720552ed7a07108f7a4bcf6148faf4fd30c13b18e22
+size 13571