Training in progress, step 1900

checkpoint-1500/latest

@@ -1 +0,0 @@
-global_step1500

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be234974669f3fadc6f6cbd37d452cd9934b5baf0c5c8c7e1c6b4137198abe52
+oid sha256:47aebfb81f8363ed05b74e6e078681b273c5c586465aaeef3d06ab2733851a41
 size 10107626487

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:efc2e8be7f1f3ab9f8f99201560b172df4044f01701bc038d6290d215cfae01c
+oid sha256:e2c4b9eee899438e56990c5b63498d9dcb5a31753aaf89888cc5b0f2ae771706
 size 10107626487

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2165ab382bfdfdbf16b93de6450522cccf99b62071bca0b967949cf333ac657e
+oid sha256:4ea0f41d07ac677f422404956c83f402b623d1e9d38f80d96aa7df1573be8ff3
 size 10107626487

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ffd46eeebd3371a3b4b86eb3043327577493233d73a94c9bb477d60b233368c
+oid sha256:ff2158c21bc75a93ba75193e8cd8dfad392a2398d8707967fcb537e61891b1a3
 size 10107626487

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:53b0718ea2367838043f31bbcecb9244b1ed91ffef1d2ca1462dd26c04c7489c
+oid sha256:c811cf5b1861c8cb662c054b12a7c96a038902adb71f04de6bf96a65b6246ec4
 size 10107626487

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54231c179632b2326d7b09b3385f441573f6500c70b342193c4260c7c8bd085e
+oid sha256:a9369b4ef2bf8dddb4168a4446f9088e8f3fb3b051aff7a7e19a3049c2070e54
 size 10107626487

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd30beca9133984f1cb9d79a189892ed188f27c8388db7aa58c1da5602b68251
+oid sha256:5ef94ff510dd0d525c7d5571350b2b024f9cab3ccce19b0a137485160166c1b8
 size 10107626487

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ffc12faa6160bd48d1e8d7c22fd5dacfb9b2c72fd9b8c0b15b28e54a7ceebad9
+oid sha256:fa3a389cbb94be741314103205c427d4a3571699efd16e0fd5f242bcb0c9f9f8
 size 10107626487

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eee19a14911d3ed28d1d543bd79d2dd862e5a570f2d967e42a87991d0a33e8c6
+oid sha256:9ccec6639f6021228a72cacd939c896abbd5ea2c92885d42b8682b9ec5808e12
 size 168086

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c3c0127bac24efc303b80e612fc9e224dd30b89da7db5488f6c14771d46648f2
+oid sha256:eb94590dee0991ff663b4bfea29d11e9fb5258d9f6b09f9dc2c4ed958a22cbbc
 size 168086

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09ee6e175062ef86f7fc01d12a5bc90ac8148ba0918a8debb63553579b33b7fa
+oid sha256:fd1a137c362ef4d5e939a77f90b7f1238a607611aca192d48e5c14e5590f89c8
 size 168086

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d50b37adf6fa6b474bee839f1d619ee75ce3c807e483c9db094f19f9db3e95f
+oid sha256:00c67c840ba12469a70530ece889168c13d1c236e3c47f561627f2fb998a5811
 size 168086

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5fa6ab2edb8180b9dd9b85477f05b17f0d390568d5145a16e8aa261e96cc1d1b
+oid sha256:c6cd53ec91a613c2b080addf94707266554f06bea53f22e5f2410638b7c2f359
 size 168086

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e330ab2aa79060a8373472b9f063fd27cab075cee334e9d1a8bb9172811b145c
+oid sha256:e8380e0444737b81d5c7bb2a3201de3b50244230313758be4830937fbfcdd62e
 size 168086

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49a42a6db08c6f1abc3d971a342c8b8af2eafd434e02783f9576a53acf666928
+oid sha256:99848a79deee2a4543bbb55739697dc1b1e83d7124d2b9100b203e0aea8ede67
 size 168086

{checkpoint-1500/global_step1500 → checkpoint-1900/global_step1900}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8251c8a3a28153639016d2e924ecd80a8c9779255ab274fcbccce7098548386c
+oid sha256:c98e12012ad590a146a8cf1c7bce43258a07f093310465ea9b884c26768dfbb4
 size 168086

checkpoint-1900/latest

@@ -0,0 +1 @@
+global_step1900

{checkpoint-1500 → checkpoint-1900}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9eb4326fec531d0679ee14fa4bdac4c9f46ba12c00419f7e9672fa8f3f3fad0a
+oid sha256:b0250158745c111e3a3bd8c0c63bb88e9f24ad6e3b56dea72fd77a43559e57cb
 size 9976576392

{checkpoint-1500 → checkpoint-1900}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37b31b0207fede92f81ca33b70043c969525893bbe5d409f4c636566178f19b9
+oid sha256:bf8e87043471e756376e41c4bb6a0d60729025052f4bc413ae38b154f3eaa9d9
 size 3500296504

{checkpoint-1500 → checkpoint-1900}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.5,
-  "global_step": 1500,
+  "epoch": 0.6333333333333333,
+  "global_step": 1900,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -906,11 +906,251 @@
       "learning_rate": 0.0003,
       "loss": 0.1118,
       "step": 1500
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0003,
+      "loss": 0.113,
+      "step": 1510
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0003,
+      "loss": 0.3687,
+      "step": 1520
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0003,
+      "loss": 0.1801,
+      "step": 1530
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0003,
+      "loss": 0.1534,
+      "step": 1540
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0003,
+      "loss": 0.1248,
+      "step": 1550
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0003,
+      "loss": 0.1091,
+      "step": 1560
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0003,
+      "loss": 0.0965,
+      "step": 1570
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0003,
+      "loss": 0.098,
+      "step": 1580
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0003,
+      "loss": 0.097,
+      "step": 1590
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0003,
+      "loss": 0.0885,
+      "step": 1600
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0003,
+      "loss": 0.0795,
+      "step": 1610
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0003,
+      "loss": 0.0843,
+      "step": 1620
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0003,
+      "loss": 0.0734,
+      "step": 1630
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0003,
+      "loss": 0.0744,
+      "step": 1640
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0003,
+      "loss": 0.0794,
+      "step": 1650
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0003,
+      "loss": 0.0708,
+      "step": 1660
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0003,
+      "loss": 0.1133,
+      "step": 1670
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0003,
+      "loss": 0.1582,
+      "step": 1680
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0003,
+      "loss": 0.1106,
+      "step": 1690
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0003,
+      "loss": 0.0904,
+      "step": 1700
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0003,
+      "loss": 0.0785,
+      "step": 1710
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0003,
+      "loss": 0.0747,
+      "step": 1720
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0003,
+      "loss": 0.0704,
+      "step": 1730
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0003,
+      "loss": 0.0634,
+      "step": 1740
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0003,
+      "loss": 0.0629,
+      "step": 1750
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0003,
+      "loss": 0.057,
+      "step": 1760
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0003,
+      "loss": 0.0563,
+      "step": 1770
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0003,
+      "loss": 0.054,
+      "step": 1780
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0003,
+      "loss": 0.0532,
+      "step": 1790
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0003,
+      "loss": 0.0525,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0003,
+      "loss": 0.0519,
+      "step": 1810
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0003,
+      "loss": 0.0478,
+      "step": 1820
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0003,
+      "loss": 0.0473,
+      "step": 1830
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0003,
+      "loss": 0.0473,
+      "step": 1840
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0003,
+      "loss": 0.0492,
+      "step": 1850
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0003,
+      "loss": 0.0451,
+      "step": 1860
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0003,
+      "loss": 0.0453,
+      "step": 1870
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0003,
+      "loss": 0.0439,
+      "step": 1880
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0003,
+      "loss": 0.0435,
+      "step": 1890
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0003,
+      "loss": 0.0435,
+      "step": 1900
     }
   ],
   "max_steps": 3000,
   "num_train_epochs": 9223372036854775807,
-  "total_flos": 628138967040000.0,
+  "total_flos": 795642691584000.0,
   "trial_name": null,
   "trial_params": null
 }

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:953ae36b2f89bfca964195d3702f47bd8b1f095dbf1e56005adcfb5972042fe7
-size 32411
+oid sha256:1542e5f72a0deb50ea56f278bcbfb04cc7ef3686cad25fabc136934d1b2ae0e6
+size 33981