Training in progress, step 700

checkpoint-300/latest

@@ -1 +0,0 @@
-global_step300

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd0659fa336d51e2aaabbcfcea7e2180c80f9ece733d53cf78b3f152d3f3f4d2
+oid sha256:657e059a4d3ebfb5ff04b25a9469094a084fe3c375ff571008871d3ecc3b9479
 size 10107626487

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:047d63e7fa4fd1897a36c8d28179ffb30a3049f2936a7ded3565f64de61f1ba0
+oid sha256:314acaa7bbeead7d30a504b5d79aa6872dee4f3c51f324ee66ed4fb593f89b9b
 size 10107626487

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d49a954428852d1406e2515f1421d7dbfb60753988a6b86641ce25194e8f5d5
+oid sha256:e2f7ea121bab4323b78306fa9218d3d46c516b848161604ade5ab75e576c454a
 size 10107626487

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a757019abf4cc50cbcbb44a26d96972cf6c5695ca2c9f19eabe937b7969c7f80
+oid sha256:494c82953b842a9447dbb3101e54cdfc7c5a74f7aa6093f34991d5e5fcadcf37
 size 10107626487

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a9f9d5976388200dde7d036c7cd383283e85938f9036d64292280d5a0b55001b
+oid sha256:e66d4b95797da073e5fbb9d29b9b03e33e233e4e19ce308b926f2386bdb18801
 size 10107626487

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:89922895f44aa86e3a687951dbca1e0efb7b80e947df9fd0c4953e07eb0502d4
+oid sha256:04bc9551489892dfa302945f623812fb47f223bff9bae2fb5fc66d2389a1fa10
 size 10107626487

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:46069972f39e02c500fe4c242149b335ac75d3eec6ac05ac74669636d191d31b
+oid sha256:d179f03c4aeb318805ea3c261a172b1ae95c4633ebd4db2d268aa1d63a273dae
 size 10107626487

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6a4fd527a2c035276df4ef90032a70a9d4489a8dbea9bc24acb40f878cb3743e
+oid sha256:2e8fd71eeba31523624dee677e38099b01affdec2733503b19390b89533c0426
 size 10107626487

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:97e9e6b53f261b9ae6a781073a2100a1acabc03bc1f225706f3a1714131e51a8
+oid sha256:95c68f4d44abdf104c4b8b9b29178513935038912b4714a9d59052f18bac6a7b
 size 168086

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f9cf16b559408c0db59839add3f495505a67f087a8dd5d8e1db5852cfaec134
+oid sha256:bbe887206a60b350ceae6a0711b7d298c97c0c04b61918dd937a185b6f3d97e9
 size 168086

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4acf9e8a7e193ea59cabf616bb6e2f9af1596025631319afb0dd94beec5959d
+oid sha256:e7be59d263a1de01b259a042057f66624d5b57dcdd10ac545e4bb4363e8c1836
 size 168086

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5bc337de13752cde49cdf482237fb6e65232f847636e00277d4b73c757d9aac7
+oid sha256:4ffc478c7310bc114b77e3c8b945b2c978792c31f17d11dce25c92f118e9a9bf
 size 168086

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:59bdba10fd52f2360a49fec01c2cc7e3f4ef2c2918bba9e71243b957452d433a
+oid sha256:9fdda703be13082a93c62f317a63cb06ef04698b23cc2d0f5989350d4e598b40
 size 168086

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9598d5d4a340626ae5b37adc9002b195d6c462d79b2c7cbc6bc534ac0a0d6428
+oid sha256:691e71f782b41b7c9160535e1bdd178f1d6312263a3f15d4236d0c59f406748c
 size 168086

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:80b56dcb40c2df2005f2cea18adfe824d1577c12f9afb846fd82012af6e82052
+oid sha256:fd11fdbd270f54d60b550cbca38c0f4a8f9512b1b309d66cb6d7fd3c157e4b51
 size 168086

{checkpoint-300/global_step300 → checkpoint-700/global_step700}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ae77ebbcd0165d630ec5ec1fcd79fc8a729eb5c4e74ed9d0dfb491af5d9af15
+oid sha256:74bb89324a7284704b916ee31044be4aaa030ad235410b0c1b975252d97223bf
 size 168086

checkpoint-700/latest

@@ -0,0 +1 @@
+global_step700

{checkpoint-300 → checkpoint-700}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:20a6a5201f9239ae7dc8ba53f8248da5a2cb5d78c6e20d80496431de61acd837
+oid sha256:0a7808980336086d17fa408bbeb87bd23f94c657f33e0b45eafa6d954c22a012
 size 9976576392

{checkpoint-300 → checkpoint-700}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2bb174aa94d762c16ee2c19f6a44d8ead2e1b4cc12bdea2b36d3cb8848fe26f3
+oid sha256:c8a472affbafffc298d8f8276dace99afdb42e63c5e8ebcab3abb9cc7349c40d
 size 3500296504

{checkpoint-300 → checkpoint-700}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.1,
-  "global_step": 300,
+  "epoch": 0.23333333333333334,
+  "global_step": 700,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -186,11 +186,251 @@
       "learning_rate": 0.0003,
       "loss": 4.695,
       "step": 300
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0003,
+      "loss": 4.6808,
+      "step": 310
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0003,
+      "loss": 4.6423,
+      "step": 320
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0003,
+      "loss": 4.613,
+      "step": 330
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0003,
+      "loss": 4.5851,
+      "step": 340
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0003,
+      "loss": 4.5882,
+      "step": 350
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0003,
+      "loss": 4.6228,
+      "step": 360
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0003,
+      "loss": 4.6269,
+      "step": 370
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0003,
+      "loss": 4.5364,
+      "step": 380
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0003,
+      "loss": 4.4992,
+      "step": 390
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0003,
+      "loss": 4.4799,
+      "step": 400
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0003,
+      "loss": 4.3733,
+      "step": 410
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0003,
+      "loss": 4.1788,
+      "step": 420
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0003,
+      "loss": 3.6706,
+      "step": 430
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0003,
+      "loss": 2.8767,
+      "step": 440
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0003,
+      "loss": 2.3927,
+      "step": 450
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0003,
+      "loss": 2.3062,
+      "step": 460
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0003,
+      "loss": 2.2299,
+      "step": 470
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0003,
+      "loss": 2.1739,
+      "step": 480
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0003,
+      "loss": 2.1303,
+      "step": 490
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0003,
+      "loss": 2.1092,
+      "step": 500
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0003,
+      "loss": 2.0715,
+      "step": 510
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0003,
+      "loss": 2.0473,
+      "step": 520
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0003,
+      "loss": 2.0566,
+      "step": 530
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0003,
+      "loss": 1.9818,
+      "step": 540
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0003,
+      "loss": 1.9592,
+      "step": 550
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0003,
+      "loss": 1.8939,
+      "step": 560
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0003,
+      "loss": 1.8441,
+      "step": 570
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0003,
+      "loss": 1.808,
+      "step": 580
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0003,
+      "loss": 1.765,
+      "step": 590
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0003,
+      "loss": 1.706,
+      "step": 600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0003,
+      "loss": 1.6328,
+      "step": 610
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003,
+      "loss": 1.6197,
+      "step": 620
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003,
+      "loss": 1.6106,
+      "step": 630
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003,
+      "loss": 1.5649,
+      "step": 640
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003,
+      "loss": 1.5001,
+      "step": 650
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003,
+      "loss": 1.4926,
+      "step": 660
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003,
+      "loss": 1.4682,
+      "step": 670
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003,
+      "loss": 1.442,
+      "step": 680
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003,
+      "loss": 1.4117,
+      "step": 690
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003,
+      "loss": 1.379,
+      "step": 700
     }
   ],
   "max_steps": 3000,
   "num_train_epochs": 9223372036854775807,
-  "total_flos": 125627793408000.0,
+  "total_flos": 293131517952000.0,
   "trial_name": null,
   "trial_params": null
 }

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:accaaa15e2b69671590fc720552ed7a07108f7a4bcf6148faf4fd30c13b18e22
-size 13571
+oid sha256:6ebc5354d2855ef417ffbf8fe045db7b97365794e8867bcd1a217a43e697f39f
+size 15141