Training in progress, step 2000

checkpoint-1600/latest

@@ -1 +0,0 @@
-global_step1600

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a340884351df9f228d3ba5317543b112e21edc4d1572228a4abd7118e419a6b
+oid sha256:b86230a2ff339a4eed9c61f759f6d801d3fae222c0ffec184fef39b202862127
 size 10107626487

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:27f75017ef2953071a44cde3f1155976fb76dde43b6395726f2683ee1ec2c250
+oid sha256:d5b8bb19172786edf6574e43a9c4b464efc2951d5c0b65458d07fa6afe9df15f
 size 10107626487

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fdfad233cd4ea82dc4f02a8e5b074984bb6a29a18f6262838ec7b7f1630e0ac8
+oid sha256:2f8d3afd87e522208354f159d281125efbbfd8595bf981ba6223257b2944354c
 size 10107626487

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5919bc8e571e808ff3b2bcb9ad7014597b2ee31e29993555c14491a33d11f095
+oid sha256:47c7a4c8faae05cd37eb1a20d114c300a5a013bdda9679c80bebf1fd441dd432
 size 10107626487

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c354eb1ceeeb29fb3a5f70328aa4850898429dd56b0f49cfb1b15ac4000f975
+oid sha256:a588583656067de4b1e87507cb2c2e1244a005b5443c82f0d75aeaa3760cdfb6
 size 10107626487

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fead0347839e08434f2eed3165dad3e70691836ea6e9cf64ef56551331997bca
+oid sha256:75a237a297019da5c7a00ad5dda55296958bc2c3ecdf3182fc2edda8a27e481a
 size 10107626487

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8ffe9fbb7a126ec39112724337caecc4eae7d8e492d63490d8567816ba07929
+oid sha256:991dcd9a7d7a82983e239fcf4412f5dc61102f27b8319a599c7d11680903f3a0
 size 10107626487

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:96e21de250ee92a08fe926befd4f33f363a7c9bd7ca8bbb6aef12bce9df04133
+oid sha256:510ff51bd274ede579cc39b4be0069bccd126470cf05c2334d71122857119a8a
 size 10107626487

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91599707db75d7b03814882cbcbb8b854875d9e88102c0831463b1e42ed49ab1
+oid sha256:6f06d86e91ff84582e5816e499e839bf1d2883228dba79604295f18fd7dc4d81
 size 168086

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6831020de473339b10b00488798cd0193af1763454bfa118f482faf07d70a44d
+oid sha256:1f902162193bb7dfeb7d489c7c1d9e7ec375c73e0052cf4393daec6d3b6c9c64
 size 168086

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ddb9abf6f8ac61756727e9fcc585fd5a1a70c1a20ca3eb82c79388ae9fe06779
+oid sha256:a13900136a358589c9e182e7d7ea628457d78ead8ac21e620ba07662be53e5fa
 size 168086

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a24b271888c79b57824d2821f84a48dcb6381041a8b0c6c06463ebd491bb032d
+oid sha256:4ce2c58d0dd9d4730b644313621a4f6f77a9f14d57207002392948064896ba6b
 size 168086

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93af96588d797250ee8a47897e3e963aa391dabb815f6714e949496001e60f03
+oid sha256:c93cd97fb74752201ba28f0a3e3726ae8e0100f95455685923fea8410ee07442
 size 168086

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:288e31b1bddda038ec9427bc5bb7b953b1d58f1e7c11dbbf18e74dc37512d377
+oid sha256:bb802fdb973c3617bf87161de7a1023dfeb05b9680bad7d2d12b9f6200cab168
 size 168086

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bab9e0ea1b5660b9e2221c0e11c6a20f979a0b1641b3514c45386900d1001699
+oid sha256:1c8c558e62de075e88c652f7102d1bdc2487792643685bd9e42b6cdbecc59342
 size 168086

{checkpoint-1600/global_step1600 → checkpoint-2000/global_step2000}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:122e850bb9d0ec838955d8d4d6eabbb7c4437e56754813e6645e6ab2a32d64c8
+oid sha256:3f43a35e30efe0e59291ccc9e8f22e871c034af657186e34c47cc9def82c0d4a
 size 168086

checkpoint-2000/latest

@@ -0,0 +1 @@
+global_step2000

{checkpoint-1600 → checkpoint-2000}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1bb8059dad24ddb21134712cb2d7c09e74afdb9c0ad8407d026104190bec0ae3
+oid sha256:b53aa1baeaa6461a12b6d0712d6ab35ff574e6e5104ae96a3c3727e12d90eabc
 size 9976576392

{checkpoint-1600 → checkpoint-2000}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf47ea3281e028d797e91e4208dafdd29c9912946218b26b9fc740a42513cda2
+oid sha256:210e45fb0fb9b67984e732f88ec7a9c4718cc45bf8c06ac1085aacf91a12f6e4
 size 3500296504

{checkpoint-1600 → checkpoint-2000}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.5333333333333333,
-  "global_step": 1600,
+  "epoch": 0.6666666666666666,
+  "global_step": 2000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -966,11 +966,251 @@
       "learning_rate": 0.0003,
       "loss": 0.0885,
       "step": 1600
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0003,
+      "loss": 0.0795,
+      "step": 1610
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0003,
+      "loss": 0.0843,
+      "step": 1620
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0003,
+      "loss": 0.0734,
+      "step": 1630
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0003,
+      "loss": 0.0744,
+      "step": 1640
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0003,
+      "loss": 0.0794,
+      "step": 1650
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0003,
+      "loss": 0.0708,
+      "step": 1660
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0003,
+      "loss": 0.1133,
+      "step": 1670
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0003,
+      "loss": 0.1582,
+      "step": 1680
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0003,
+      "loss": 0.1106,
+      "step": 1690
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0003,
+      "loss": 0.0904,
+      "step": 1700
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0003,
+      "loss": 0.0785,
+      "step": 1710
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0003,
+      "loss": 0.0747,
+      "step": 1720
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0003,
+      "loss": 0.0704,
+      "step": 1730
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0003,
+      "loss": 0.0634,
+      "step": 1740
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0003,
+      "loss": 0.0629,
+      "step": 1750
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0003,
+      "loss": 0.057,
+      "step": 1760
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0003,
+      "loss": 0.0563,
+      "step": 1770
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0003,
+      "loss": 0.054,
+      "step": 1780
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0003,
+      "loss": 0.0532,
+      "step": 1790
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0003,
+      "loss": 0.0525,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0003,
+      "loss": 0.0519,
+      "step": 1810
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0003,
+      "loss": 0.0478,
+      "step": 1820
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0003,
+      "loss": 0.0473,
+      "step": 1830
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0003,
+      "loss": 0.0473,
+      "step": 1840
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0003,
+      "loss": 0.0492,
+      "step": 1850
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0003,
+      "loss": 0.0451,
+      "step": 1860
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0003,
+      "loss": 0.0453,
+      "step": 1870
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0003,
+      "loss": 0.0439,
+      "step": 1880
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0003,
+      "loss": 0.0435,
+      "step": 1890
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0003,
+      "loss": 0.0435,
+      "step": 1900
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0003,
+      "loss": 0.0412,
+      "step": 1910
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0003,
+      "loss": 0.0441,
+      "step": 1920
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0003,
+      "loss": 0.04,
+      "step": 1930
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0003,
+      "loss": 0.041,
+      "step": 1940
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0003,
+      "loss": 0.1391,
+      "step": 1950
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0003,
+      "loss": 0.1502,
+      "step": 1960
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0003,
+      "loss": 0.0767,
+      "step": 1970
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0003,
+      "loss": 0.0563,
+      "step": 1980
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0003,
+      "loss": 0.0487,
+      "step": 1990
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0003,
+      "loss": 0.042,
+      "step": 2000
     }
   ],
   "max_steps": 3000,
   "num_train_epochs": 9223372036854775807,
-  "total_flos": 670014898176000.0,
+  "total_flos": 837518622720000.0,
   "trial_name": null,
   "trial_params": null
 }

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1542e5f72a0deb50ea56f278bcbfb04cc7ef3686cad25fabc136934d1b2ae0e6
-size 33981
+oid sha256:140ad3bdd050ce33f869362645d2fe35a1d764c22645cccd1674c2d206ead24c
+size 35551