Training in progress, step 1600

checkpoint-1200/latest

@@ -1 +0,0 @@
-global_step1200

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4fb262812656b3c83b6881b1dd67c5767d8042011235ae6468454646240b5c07
+oid sha256:1a340884351df9f228d3ba5317543b112e21edc4d1572228a4abd7118e419a6b
 size 10107626487

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7df89141b87c7b3401cf3dc6d0f0e35b07ee4007b03d264482fa67df22e6e386
+oid sha256:27f75017ef2953071a44cde3f1155976fb76dde43b6395726f2683ee1ec2c250
 size 10107626487

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:304104b8210689a208414ce338a1a8170d07fd50ede690c06cc17f711af7d23b
+oid sha256:fdfad233cd4ea82dc4f02a8e5b074984bb6a29a18f6262838ec7b7f1630e0ac8
 size 10107626487

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:11e7a70ad03df41c4584486b35cde5c60d0c66a22c3f5c4048017d438d1d7dd5
+oid sha256:5919bc8e571e808ff3b2bcb9ad7014597b2ee31e29993555c14491a33d11f095
 size 10107626487

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:11e5c69f087a06dfa4218e7205637830d59e2eb6ab4496e20b6099fcccbc9b07
+oid sha256:8c354eb1ceeeb29fb3a5f70328aa4850898429dd56b0f49cfb1b15ac4000f975
 size 10107626487

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37dd48878dd48bb7d2a1f2ea052444d84a13e6f4cbab8f1d27904757ae116bd0
+oid sha256:fead0347839e08434f2eed3165dad3e70691836ea6e9cf64ef56551331997bca
 size 10107626487

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d614936212979e10ebba5d41469df3f0f5d966c2da7d3029cfa1ec1e9dd897a
+oid sha256:f8ffe9fbb7a126ec39112724337caecc4eae7d8e492d63490d8567816ba07929
 size 10107626487

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3266f649233e5304749c7c1f07769559935f00b03922e28bff0442991ec9388a
+oid sha256:96e21de250ee92a08fe926befd4f33f363a7c9bd7ca8bbb6aef12bce9df04133
 size 10107626487

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee838bbe2d8ed873c80d9a5740fcb7f54c91e439e0f4bdde882b4456ea232c7c
+oid sha256:91599707db75d7b03814882cbcbb8b854875d9e88102c0831463b1e42ed49ab1
 size 168086

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4326e6a49ee2354fb513f4f301a33c21ffb84a9a51a4e3d0ed33bc12d02bd10b
+oid sha256:6831020de473339b10b00488798cd0193af1763454bfa118f482faf07d70a44d
 size 168086

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d6c42d8a6eeae2786121f541dc5d4a06728de74fdd785c4e82bf7be14708e787
+oid sha256:ddb9abf6f8ac61756727e9fcc585fd5a1a70c1a20ca3eb82c79388ae9fe06779
 size 168086

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca8f493f0fa1429963ac77a6e815a9f189220490b4f4457f7be2b942cdc01726
+oid sha256:a24b271888c79b57824d2821f84a48dcb6381041a8b0c6c06463ebd491bb032d
 size 168086

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c1911bcfb001e09fe12e3ee69beaea09f5f021dc647d8b1040b0d66821cc448
+oid sha256:93af96588d797250ee8a47897e3e963aa391dabb815f6714e949496001e60f03
 size 168086

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c7c6cda3f5f3d82146b47b1ad4b836ecfff16ca2e586ea073dd665626f8757e
+oid sha256:288e31b1bddda038ec9427bc5bb7b953b1d58f1e7c11dbbf18e74dc37512d377
 size 168086

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:23cb97ab9d849ada9216fd67b47e13c5ebfbeaeb019c59c09a9cb3298aa097e6
+oid sha256:bab9e0ea1b5660b9e2221c0e11c6a20f979a0b1641b3514c45386900d1001699
 size 168086

{checkpoint-1200/global_step1200 → checkpoint-1600/global_step1600}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5bb17570ce5f68b67d4232815d8eb3b18765ca6071753b7ef99040ac8710b3ec
+oid sha256:122e850bb9d0ec838955d8d4d6eabbb7c4437e56754813e6645e6ab2a32d64c8
 size 168086

checkpoint-1600/latest

@@ -0,0 +1 @@
+global_step1600

{checkpoint-1200 → checkpoint-1600}/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:050176f4baa1bd8c33427c5122e69e830566029acf7a4cca7415b3b41047fcbd
+oid sha256:1bb8059dad24ddb21134712cb2d7c09e74afdb9c0ad8407d026104190bec0ae3
 size 9976576392

{checkpoint-1200 → checkpoint-1600}/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:30aa4a97478659826d8dbb57cb3e6ba545204647306edfcfec56c36a5b191c85
+oid sha256:bf47ea3281e028d797e91e4208dafdd29c9912946218b26b9fc740a42513cda2
 size 3500296504

{checkpoint-1200 → checkpoint-1600}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.4,
-  "global_step": 1200,
+  "epoch": 0.5333333333333333,
+  "global_step": 1600,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -726,11 +726,251 @@
       "learning_rate": 0.0003,
       "loss": 0.3855,
       "step": 1200
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003,
+      "loss": 0.3662,
+      "step": 1210
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003,
+      "loss": 0.3727,
+      "step": 1220
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003,
+      "loss": 0.3563,
+      "step": 1230
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003,
+      "loss": 0.3297,
+      "step": 1240
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0003,
+      "loss": 0.314,
+      "step": 1250
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0003,
+      "loss": 0.2996,
+      "step": 1260
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0003,
+      "loss": 0.3,
+      "step": 1270
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0003,
+      "loss": 0.2773,
+      "step": 1280
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0003,
+      "loss": 0.2665,
+      "step": 1290
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0003,
+      "loss": 0.2646,
+      "step": 1300
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0003,
+      "loss": 0.2406,
+      "step": 1310
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0003,
+      "loss": 0.2206,
+      "step": 1320
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0003,
+      "loss": 0.2325,
+      "step": 1330
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0003,
+      "loss": 0.2152,
+      "step": 1340
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0003,
+      "loss": 0.2011,
+      "step": 1350
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0003,
+      "loss": 0.1953,
+      "step": 1360
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0003,
+      "loss": 0.1862,
+      "step": 1370
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0003,
+      "loss": 0.1814,
+      "step": 1380
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0003,
+      "loss": 0.1677,
+      "step": 1390
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0003,
+      "loss": 0.1657,
+      "step": 1400
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0003,
+      "loss": 0.155,
+      "step": 1410
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0003,
+      "loss": 0.1517,
+      "step": 1420
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0003,
+      "loss": 0.1481,
+      "step": 1430
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0003,
+      "loss": 0.1396,
+      "step": 1440
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0003,
+      "loss": 0.1301,
+      "step": 1450
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0003,
+      "loss": 0.1274,
+      "step": 1460
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0003,
+      "loss": 0.1244,
+      "step": 1470
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0003,
+      "loss": 0.1172,
+      "step": 1480
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0003,
+      "loss": 0.1177,
+      "step": 1490
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0003,
+      "loss": 0.1118,
+      "step": 1500
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0003,
+      "loss": 0.113,
+      "step": 1510
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0003,
+      "loss": 0.3687,
+      "step": 1520
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0003,
+      "loss": 0.1801,
+      "step": 1530
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0003,
+      "loss": 0.1534,
+      "step": 1540
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0003,
+      "loss": 0.1248,
+      "step": 1550
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0003,
+      "loss": 0.1091,
+      "step": 1560
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0003,
+      "loss": 0.0965,
+      "step": 1570
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0003,
+      "loss": 0.098,
+      "step": 1580
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0003,
+      "loss": 0.097,
+      "step": 1590
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0003,
+      "loss": 0.0885,
+      "step": 1600
     }
   ],
   "max_steps": 3000,
   "num_train_epochs": 9223372036854775807,
-  "total_flos": 502511173632000.0,
+  "total_flos": 670014898176000.0,
   "trial_name": null,
   "trial_params": null
 }

runs/Aug22_18-42-03_ip-26-0-150-12/events.out.tfevents.1692729850.ip-26-0-150-12.2895584.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:315e08b502208730f4cc5abee6046d569656431e4bc2fa327e702fbff33f5015
-size 27701
+oid sha256:677d38e5d0f2898fe52eabe578b18234ab0985e274317f79127ce656648b82b9
+size 29271