Training in progress, step 1100

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:657e059a4d3ebfb5ff04b25a9469094a084fe3c375ff571008871d3ecc3b9479
+oid sha256:910192e566724dee1fc1dc6987f1c77385476d324f0f81bd9af23f2113e080a7
 size 10107626487

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:314acaa7bbeead7d30a504b5d79aa6872dee4f3c51f324ee66ed4fb593f89b9b
+oid sha256:61f1e1b41f8262e168d6cbabaee5ff906517a3ee8a2ead7bd5be13778b3301f7
 size 10107626487

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2f7ea121bab4323b78306fa9218d3d46c516b848161604ade5ab75e576c454a
+oid sha256:67ac954f2ed6ed05539aa95ae3231713b4fbe699591b84cc94c82f2aede2dd49
 size 10107626487

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:494c82953b842a9447dbb3101e54cdfc7c5a74f7aa6093f34991d5e5fcadcf37
+oid sha256:c9e49f162f388cf270931b7aa65b0b9c9aaf6cdb5d40302d91dd9f0e1ba295c2
 size 10107626487

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e66d4b95797da073e5fbb9d29b9b03e33e233e4e19ce308b926f2386bdb18801
+oid sha256:1161a763f360f4a98ce6277167582207ee6fc9fd9c8b5abdfd5a209c4c49634a
 size 10107626487

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04bc9551489892dfa302945f623812fb47f223bff9bae2fb5fc66d2389a1fa10
+oid sha256:f85ec6277b7860a4f2a0d7f64f8fe82fe641343c7c41b1c42929dd3015697314
 size 10107626487

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d179f03c4aeb318805ea3c261a172b1ae95c4633ebd4db2d268aa1d63a273dae
+oid sha256:67ec4562e79680723dc7ee7b0ac2cabacdb1efa6607963f2a994942598838e21
 size 10107626487

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e8fd71eeba31523624dee677e38099b01affdec2733503b19390b89533c0426
+oid sha256:3bb6032fd72cafe86e8fcd99c359f85d4fd4f85e7374f13d9fc1a24aff2df188
 size 10107626487

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95c68f4d44abdf104c4b8b9b29178513935038912b4714a9d59052f18bac6a7b
+oid sha256:a022971e5382ee392191270920eba4792f2d4ec5585037c1fe9fa9ff54df8095
 size 168086

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bbe887206a60b350ceae6a0711b7d298c97c0c04b61918dd937a185b6f3d97e9
+oid sha256:a21a722f8a14282dbc56a326a8a759992cd5dd28f453e399cf89d81dcb339ea6
 size 168086

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e7be59d263a1de01b259a042057f66624d5b57dcdd10ac545e4bb4363e8c1836
+oid sha256:46635e643d7c019b78e3565364f7bee809ef8e7197b5b9d0dea7d0c710b12bcb
 size 168086

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ffc478c7310bc114b77e3c8b945b2c978792c31f17d11dce25c92f118e9a9bf
+oid sha256:0e4598ed421dbc4c3cb011d8de423da9683ea355a7f6e75d7dacfee0e89fe11f
 size 168086

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9fdda703be13082a93c62f317a63cb06ef04698b23cc2d0f5989350d4e598b40
+oid sha256:5026560a0268883cf2ae7d43e32c2fcefca6cb057d4c730a8be77136302d3373
 size 168086

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:691e71f782b41b7c9160535e1bdd178f1d6312263a3f15d4236d0c59f406748c
+oid sha256:e86d87f88e26bd1ce11f9d7be3e75e20e283fe0db345548afaaa99dceed96052
 size 168086

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd11fdbd270f54d60b550cbca38c0f4a8f9512b1b309d66cb6d7fd3c157e4b51
+oid sha256:db3f7202cfede94534d5498f63e012ca1900f8ac02b249550b40fb836d5e9be8
 size 168086

{checkpoint-700/global_step700 → checkpoint-1100/global_step1100}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:74bb89324a7284704b916ee31044be4aaa030ad235410b0c1b975252d97223bf
+oid sha256:4c59f381ff19b5347b25e41a80384718e30708c99755dbe2c66f53e3d7dbef70
 size 168086

checkpoint-1100/latest

@@ -0,0 +1 @@
+global_step1100

{checkpoint-700 → checkpoint-1100}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a7808980336086d17fa408bbeb87bd23f94c657f33e0b45eafa6d954c22a012
+oid sha256:e9e589f280307fec92df9299851e14587c937700c97acd438840b30e79b1f025
 size 9976576392

{checkpoint-700 → checkpoint-1100}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8a472affbafffc298d8f8276dace99afdb42e63c5e8ebcab3abb9cc7349c40d
+oid sha256:2561123382fff632d42a7bd0b76899385470af4cf48b4c423bfcccdec12b6205
 size 3500296504

{checkpoint-700 → checkpoint-1100}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.23333333333333334,
-  "global_step": 700,
+  "epoch": 0.36666666666666664,
+  "global_step": 1100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -426,11 +426,251 @@
       "learning_rate": 0.0003,
       "loss": 1.379,
       "step": 700
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003,
+      "loss": 1.3609,
+      "step": 710
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003,
+      "loss": 1.3224,
+      "step": 720
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003,
+      "loss": 1.2853,
+      "step": 730
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003,
+      "loss": 1.2762,
+      "step": 740
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003,
+      "loss": 1.2265,
+      "step": 750
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003,
+      "loss": 1.2232,
+      "step": 760
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003,
+      "loss": 1.2075,
+      "step": 770
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003,
+      "loss": 1.168,
+      "step": 780
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003,
+      "loss": 1.1472,
+      "step": 790
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003,
+      "loss": 1.118,
+      "step": 800
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003,
+      "loss": 1.1228,
+      "step": 810
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003,
+      "loss": 1.1339,
+      "step": 820
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003,
+      "loss": 1.0853,
+      "step": 830
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003,
+      "loss": 1.0676,
+      "step": 840
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003,
+      "loss": 1.0905,
+      "step": 850
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003,
+      "loss": 1.076,
+      "step": 860
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003,
+      "loss": 1.0202,
+      "step": 870
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003,
+      "loss": 1.0123,
+      "step": 880
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003,
+      "loss": 0.9863,
+      "step": 890
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003,
+      "loss": 0.9347,
+      "step": 900
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003,
+      "loss": 0.9416,
+      "step": 910
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003,
+      "loss": 0.9165,
+      "step": 920
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003,
+      "loss": 0.8996,
+      "step": 930
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003,
+      "loss": 0.8673,
+      "step": 940
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003,
+      "loss": 0.8449,
+      "step": 950
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003,
+      "loss": 0.8468,
+      "step": 960
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003,
+      "loss": 0.817,
+      "step": 970
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003,
+      "loss": 0.7947,
+      "step": 980
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003,
+      "loss": 0.7706,
+      "step": 990
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003,
+      "loss": 0.7357,
+      "step": 1000
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003,
+      "loss": 0.6983,
+      "step": 1010
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003,
+      "loss": 0.6989,
+      "step": 1020
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003,
+      "loss": 0.6692,
+      "step": 1030
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003,
+      "loss": 0.6553,
+      "step": 1040
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003,
+      "loss": 0.6272,
+      "step": 1050
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003,
+      "loss": 0.6092,
+      "step": 1060
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003,
+      "loss": 0.604,
+      "step": 1070
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003,
+      "loss": 0.6494,
+      "step": 1080
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003,
+      "loss": 0.6155,
+      "step": 1090
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003,
+      "loss": 0.5713,
+      "step": 1100
     }
   ],
   "max_steps": 3000,
   "num_train_epochs": 9223372036854775807,
-  "total_flos": 293131517952000.0,
+  "total_flos": 460635242496000.0,
   "trial_name": null,
   "trial_params": null
 }

checkpoint-700/latest

@@ -1 +0,0 @@
-global_step700

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1fef6847951e7a7a5d7472c77aeb6d8f614b223d630dfc7f7950ed07e82dfab5
-size 19851
+oid sha256:9195eb0ab07f76500bcac0b0eec3ca51997f05cae782877112fd61a0f25f510c
+size 21421