Training in progress, step 2100

checkpoint-1700/latest

@@ -1 +0,0 @@
-global_step1700

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:739358ce4d0db4ffe39ed9682bd0746d64e877475f82b11cc197a2d1776a95bf
+oid sha256:b5e17adae64d158ed7b80018b62d4f64f76a717739f3667ab868bd08b5546e36
 size 10107626487

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eddeb7918d9da8d945eff4226bad6433fb778ad8ac595d07ec9e6b727cc1fc88
+oid sha256:90103633ebee5c418c077e4b0d6fc4c2936161788ab990a5b63b49058bf980cd
 size 10107626487

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be571f757a3b36b7770415b18d750383e66b8f61adee886e8fd35b9f511d00a5
+oid sha256:ac402f4e9dfd0a9cdeedc219939c8d2854a425000535a845a391b5a61be5fa21
 size 10107626487

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8f82328ed0b0f3007255807c4af4a3f22fc4a6160d94d1fcab8c2a6cf33febf
+oid sha256:0587a206f02896afd81610085e44cc69b3fd43d70cc64144dda25c7471d6c721
 size 10107626487

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:38205fee9c0d3d5fe1f62fed51614a5f8494d171cbf3a9980100ae29b0455e2f
+oid sha256:d431dee2fe9c6c5124ad116f289b7ae04ae192823760eb75d95eb02f500ec778
 size 10107626487

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:494e2e6979df9d5a6e1c501ca19a5d391eb72b5b7e453ef352b99c95af7e30c0
+oid sha256:de5485088c9b41156808d8346e4617be253fea5de5cd5d4b91f71a26e2e24ea1
 size 10107626487

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:acd19a1452e5c563fdb4431f7f48bbae00d8854ca2069793f4fbc0beceda5cf1
+oid sha256:5eed860602ba38c80b647d3f2412ee1d6b4778dbfce7131878373bbfbacb187c
 size 10107626487

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e74f03870306d16790aa672096848730b1af5bba987a74a58a386247c7d46d0a
+oid sha256:2a484f7976af13384e667021a7316ea630924e973e29a67120db984ff15076c2
 size 10107626487

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b0de3709868caed6aec7042aa13cc5ff5a7cdecddaf21ef01d70778485299e0
+oid sha256:11aac4363e618fa55c62977722ae406ff69bb54c12e53e2fdeffa5f1632855c8
 size 168086

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:289994d8f0d45c07b722e692498d3d2b48d42f6d5773734bc0b1427168bcb03c
+oid sha256:4dfa5bde2a410bd133271a27c7be77ad69ce3015fcdc26b7f58ff280eea1d359
 size 168086

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:580fb5ca74492c6b71ff5b8bce34f5d31d862c3f96df7887c8a48b3f6241d58d
+oid sha256:2acccdd6e28633f3c199afe57b8d9b80e0d128bae92176d2309e4102e3de50e0
 size 168086

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa2370bff44b992e28f59a7625c475f616252f1a45d67c27bf62dfef50a94e25
+oid sha256:2ce5abe052a95d79d00ceac0272d90efa7b18a68a220d8345df706b98c600cf1
 size 168086

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:122914f6c795eec77e2d9459857b6ba0d3f66a9e407f4ac2889abd3c8353699a
+oid sha256:59a2ce519ef75a9502353294f95984f83b8c1473ae7ea64a1828949d6d3e0e41
 size 168086

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95c4796941ff69c9089277ccc210ed66a53aa2aa27fd08592f1477f03ff51105
+oid sha256:851aceef9941342286cbd797e3b61362e8d088deabe520a249a37e2be273cc80
 size 168086

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d83660c9172cbdd43bb2f7680c46745082730200dd6e9ca742ce0059c3df8ee
+oid sha256:cbc4e5fa62b4b71b517895a983df2fc1b618aef52984e9caa8fe7898198a2e10
 size 168086

{checkpoint-1700/global_step1700 → checkpoint-2100/global_step2100}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:31494943d94985534fd92f9d7ee175a61321dda3900fc4acafdd4b55c8335f81
+oid sha256:e40d7b508c8727eadc3ef7115d9b998796e9f503c498fdea49536a01996484fe
 size 168086

checkpoint-2100/latest

@@ -0,0 +1 @@
+global_step2100

{checkpoint-1700 → checkpoint-2100}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e03a32d4279f008ea227183bce3bea02bdc787efdefe57a690ce1bce74dce1e1
+oid sha256:6adc6b1233e38d5ffa6e2dd4bc49283f9ba67f0e6796f4957275ba8eac5007fa
 size 9976576392

{checkpoint-1700 → checkpoint-2100}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9ff5280ad459b4473324fb3ecebb46301b963d7f24d4e3457da984689305500
+oid sha256:fb05fce5d6454ad204985cd2264d190114a5a59a137bd596e9372925ffd56913
 size 3500296504

{checkpoint-1700 → checkpoint-2100}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.5666666666666667,
-  "global_step": 1700,
+  "epoch": 0.7,
+  "global_step": 2100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1026,11 +1026,251 @@
       "learning_rate": 0.0003,
       "loss": 0.0904,
       "step": 1700
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0003,
+      "loss": 0.0785,
+      "step": 1710
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0003,
+      "loss": 0.0747,
+      "step": 1720
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0003,
+      "loss": 0.0704,
+      "step": 1730
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0003,
+      "loss": 0.0634,
+      "step": 1740
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0003,
+      "loss": 0.0629,
+      "step": 1750
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0003,
+      "loss": 0.057,
+      "step": 1760
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0003,
+      "loss": 0.0563,
+      "step": 1770
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0003,
+      "loss": 0.054,
+      "step": 1780
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0003,
+      "loss": 0.0532,
+      "step": 1790
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0003,
+      "loss": 0.0525,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0003,
+      "loss": 0.0519,
+      "step": 1810
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0003,
+      "loss": 0.0478,
+      "step": 1820
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0003,
+      "loss": 0.0473,
+      "step": 1830
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0003,
+      "loss": 0.0473,
+      "step": 1840
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0003,
+      "loss": 0.0492,
+      "step": 1850
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0003,
+      "loss": 0.0451,
+      "step": 1860
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0003,
+      "loss": 0.0453,
+      "step": 1870
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0003,
+      "loss": 0.0439,
+      "step": 1880
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0003,
+      "loss": 0.0435,
+      "step": 1890
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0003,
+      "loss": 0.0435,
+      "step": 1900
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0003,
+      "loss": 0.0412,
+      "step": 1910
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0003,
+      "loss": 0.0441,
+      "step": 1920
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0003,
+      "loss": 0.04,
+      "step": 1930
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0003,
+      "loss": 0.041,
+      "step": 1940
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0003,
+      "loss": 0.1391,
+      "step": 1950
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0003,
+      "loss": 0.1502,
+      "step": 1960
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0003,
+      "loss": 0.0767,
+      "step": 1970
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0003,
+      "loss": 0.0563,
+      "step": 1980
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0003,
+      "loss": 0.0487,
+      "step": 1990
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0003,
+      "loss": 0.042,
+      "step": 2000
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0003,
+      "loss": 0.0388,
+      "step": 2010
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0003,
+      "loss": 0.0424,
+      "step": 2020
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0003,
+      "loss": 0.0413,
+      "step": 2030
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0003,
+      "loss": 0.036,
+      "step": 2040
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0003,
+      "loss": 0.0333,
+      "step": 2050
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0003,
+      "loss": 0.0334,
+      "step": 2060
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0003,
+      "loss": 0.0298,
+      "step": 2070
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0003,
+      "loss": 0.0302,
+      "step": 2080
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0003,
+      "loss": 0.0302,
+      "step": 2090
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0003,
+      "loss": 0.0293,
+      "step": 2100
     }
   ],
   "max_steps": 3000,
   "num_train_epochs": 9223372036854775807,
-  "total_flos": 711890829312000.0,
+  "total_flos": 879394553856000.0,
   "trial_name": null,
   "trial_params": null
 }

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:140ad3bdd050ce33f869362645d2fe35a1d764c22645cccd1674c2d206ead24c
-size 35551
+oid sha256:b3476f29a411313ed060c01b35f1409f573b4dc1eb8ce083996033c57780434b
+size 37121