Training in progress, step 1800

checkpoint-1400/latest

@@ -1 +0,0 @@
-global_step1400

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d48928da7236975b7d563675900115bd40723d9bb946a5e12aeb3802cda65a29
+oid sha256:65ac051edb6a7231e7ff80bf7b6ff872456d98c899aa44c7f0de57ec100e241a
 size 10107626487

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:60d0660e0465b031017aa8ac057dadaad3fefbbe9a853e985e3fbcc3b9e3d14b
+oid sha256:f63f4158c79b9097827e52efe21c448978712c950ba30906df8d4e81779f0705
 size 10107626487

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc3b54faf82a144d566ec1e13391ced58d82ba950098490d54bc7e6c12b2f665
+oid sha256:a06a42dbae3be95f0f13a0c6a7e3d8fff63f06a1b88e67198a327c7c3f2b4fdf
 size 10107626487

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8730071eb5352476076fecea2c16332b76414b1587e9170294cd8be64d658ae4
+oid sha256:a2d439a4859cf52b20fb93dc23b4dba8c8083d3d58b8bd2b35129a82e4f9847f
 size 10107626487

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e1ecb36573935c513134e905740b8404414b0756f654afb39ce67f9eed0f089c
+oid sha256:56e9a0ca5c754b3520440fc3506fe5a4e46f2a2eade8194719f6c07e5ee7b32e
 size 10107626487

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2977e65540cd59b6e490d355dee2caa45edce69df4ccad7d08cd12e64717c58d
+oid sha256:ea9a156039d51fadfd2736deea73e96bf8dfb0dbf11a6c3b10eb075a358c9b8f
 size 10107626487

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a91e4e9224ed4db284c54c1c60b66d8d8db36f96ef264a6d0724d17d6d4159e2
+oid sha256:103ef36796553ded7aa4fea4a7e84fb84384939f9f20cda0a94ecf5087a56b7d
 size 10107626487

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88a2ea44cd053990b0f804df2960b863b7ee45e2d96229c63aeb1b4f124b33f1
+oid sha256:478d4c670c95787a5e189a35cf100f0c722538933effcc1efdf43d7a3a79ec87
 size 10107626487

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b427970d78d97a2990a387d0111436e57780839b349873130d3acb3276e6590
+oid sha256:1a3559441b3540dc1b5a49a8fb7253c34731a3906e17ecc6f79b18ee56a4ba0e
 size 168086

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:25cd95fea28ed0b382448d3984a3b9c57253251be9b58ca7a14bdb60e0044800
+oid sha256:2c7f165ac77d9d755737a9b47099cbc30144ddc12bbdf2a30877c884fe0aa613
 size 168086

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a12da95cea77c0e4bf6c95cf547c52358a1a3dcc47d277ebc6a798c9caa01814
+oid sha256:1d7943fb9f5e3008adcd3ed98b51af946f70d290a7337c4ffbe0eb7a9b97be6c
 size 168086

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:883c364ce237d2eaae2ccaffd1020d9bf975a0709445f76ec935f476a9e0355c
+oid sha256:734fbce19436e77d545003a57bdb0fcc5fbd70f67630be95f23d1f26b813fc8b
 size 168086

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f7dee3b066702da3217434a6fdf3be471ea2359e4a7f550407604a5b0eac806
+oid sha256:fb7a3c3a029b300ee81735e4b5d95db67561d27acbcc726de134c1749f3d5442
 size 168086

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:144d70ba74905a12764aa691b542b267d9b7798360cf8174acfb33c6916e7df6
+oid sha256:afac81b2fa8bcebadbac9d41f1d41c618419c5aa49f521a8b868d88affb1b81b
 size 168086

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c9ff5c63b0581bc3bf5296b9188a1bf794ee1128efd17617a1b6d6c5bd71ef6
+oid sha256:0c36b334041e232c5d22af305a776884cf9a86554d427f772411f35d878d40e0
 size 168086

{checkpoint-1400/global_step1400 → checkpoint-1800/global_step1800}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ddbafdcfd970d545721c1221e491df7ab9c50322e6312b7ec1b036b6481b3dcb
+oid sha256:4741731558644f602064546705ffa83f9879c68a3314de93ea623d611af243e5
 size 168086

checkpoint-1800/latest

@@ -0,0 +1 @@
+global_step1800

{checkpoint-1400 → checkpoint-1800}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d81d64070bf6ada487dbdc2aaec06e52c932926918d08c4298cb20772e593337
+oid sha256:cc8b8d03a8fe3e6b5c85db79b65e4586478a90485d98555e56952666cdae15e5
 size 9976576392

{checkpoint-1400 → checkpoint-1800}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df84e357f53f857462e754af1e9a7f50bdbf8395868f7a7403b6b72c5d94b5a7
+oid sha256:ad1f107946442c1ba1bd23d3ab4b3b2906ea95617d47c6b2f8fe6b989084b50d
 size 3500296504

{checkpoint-1400 → checkpoint-1800}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.4666666666666667,
-  "global_step": 1400,
+  "epoch": 0.6,
+  "global_step": 1800,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -846,11 +846,251 @@
       "learning_rate": 0.0003,
       "loss": 0.1657,
       "step": 1400
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0003,
+      "loss": 0.155,
+      "step": 1410
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0003,
+      "loss": 0.1517,
+      "step": 1420
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0003,
+      "loss": 0.1481,
+      "step": 1430
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0003,
+      "loss": 0.1396,
+      "step": 1440
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0003,
+      "loss": 0.1301,
+      "step": 1450
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0003,
+      "loss": 0.1274,
+      "step": 1460
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0003,
+      "loss": 0.1244,
+      "step": 1470
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0003,
+      "loss": 0.1172,
+      "step": 1480
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0003,
+      "loss": 0.1177,
+      "step": 1490
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0003,
+      "loss": 0.1118,
+      "step": 1500
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0003,
+      "loss": 0.113,
+      "step": 1510
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0003,
+      "loss": 0.3687,
+      "step": 1520
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0003,
+      "loss": 0.1801,
+      "step": 1530
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0003,
+      "loss": 0.1534,
+      "step": 1540
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0003,
+      "loss": 0.1248,
+      "step": 1550
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0003,
+      "loss": 0.1091,
+      "step": 1560
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0003,
+      "loss": 0.0965,
+      "step": 1570
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0003,
+      "loss": 0.098,
+      "step": 1580
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0003,
+      "loss": 0.097,
+      "step": 1590
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0003,
+      "loss": 0.0885,
+      "step": 1600
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0003,
+      "loss": 0.0795,
+      "step": 1610
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0003,
+      "loss": 0.0843,
+      "step": 1620
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0003,
+      "loss": 0.0734,
+      "step": 1630
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0003,
+      "loss": 0.0744,
+      "step": 1640
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0003,
+      "loss": 0.0794,
+      "step": 1650
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0003,
+      "loss": 0.0708,
+      "step": 1660
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0003,
+      "loss": 0.1133,
+      "step": 1670
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0003,
+      "loss": 0.1582,
+      "step": 1680
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0003,
+      "loss": 0.1106,
+      "step": 1690
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0003,
+      "loss": 0.0904,
+      "step": 1700
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0003,
+      "loss": 0.0785,
+      "step": 1710
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0003,
+      "loss": 0.0747,
+      "step": 1720
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0003,
+      "loss": 0.0704,
+      "step": 1730
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0003,
+      "loss": 0.0634,
+      "step": 1740
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0003,
+      "loss": 0.0629,
+      "step": 1750
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0003,
+      "loss": 0.057,
+      "step": 1760
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0003,
+      "loss": 0.0563,
+      "step": 1770
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0003,
+      "loss": 0.054,
+      "step": 1780
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0003,
+      "loss": 0.0532,
+      "step": 1790
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0003,
+      "loss": 0.0525,
+      "step": 1800
     }
   ],
   "max_steps": 3000,
   "num_train_epochs": 9223372036854775807,
-  "total_flos": 586263035904000.0,
+  "total_flos": 753766760448000.0,
   "trial_name": null,
   "trial_params": null
 }

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:959f335ea05d3561ef4372cd1e8d8c2c8124e9a070658a308f82975d8ed3a93b
-size 30841
+oid sha256:953ae36b2f89bfca964195d3702f47bd8b1f095dbf1e56005adcfb5972042fe7
+size 32411