Outer Step 4. Inner Step 5. Batch Size 336

config.json

@@ -170,7 +170,7 @@
     "247": "NON_PARTICIPATING",
     "248": "NON_PARTICIPATING",
     "249": "NON_PARTICIPATING",
-    "25": "NON_PARTICIPATING",
+    "25": "SUCCESS",
     "250": "NON_PARTICIPATING",
     "251": "NON_PARTICIPATING",
     "252": "NON_PARTICIPATING",
@@ -268,25 +268,25 @@
     "AutoModelForCausalLM": "distributed/optimized-gpt2-500m--modeling_gpt_optimized.GPTOptim"
   },
   "block_list": [
-    5148917,
-    5148921,
-    5148925,
-    5148929,
-    5148933,
-    5148938,
-    5148942,
-    5148946,
-    5148951,
-    5148955
+    5148960,
+    5149021,
+    5149026,
+    5149030,
+    5149034,
+    5149038,
+    5149042,
+    5149046,
+    5149050,
+    5149054
   ],
   "block_size": 1024,
   "bos_token_id": 50256,
   "embd_pdrop": 0.1,
   "eos_token_id": 50256,
   "initializer_range": 0.02,
-  "inner_step": 15,
+  "inner_step": 5,
   "inner_steps": 0,
-  "last_allreduce_block": 5148769,
+  "last_allreduce_block": 5148895,
   "layer_norm_epsilon": 1e-05,
   "model_type": "gpt_optimized",
   "n_embd": 1280,

inner_optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dca660e596251a51ce60e31c47a5c6f6f45899815731ff430a2b2c154be13b83
+oid sha256:0f07cad039b19f9808d8b80424f3dbc9468e8de30b563f8979113f9447fe52bb
 size 8081781770

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4c2c0ca6c9528adc4bd21f46d5031b35cdf015a2b5fc819181afeebc1a1bf76a
+oid sha256:3bb5cefdfc38e1fee11a2449c2c206918171122775f9581396f5908c6cdf9fb3
 size 4040701744