Outer Step 3. Inner Step 15. Batch Size 96

config.json

@@ -268,23 +268,23 @@
     "AutoModelForCausalLM": "distributed/optimized-gpt2-500m--modeling_gpt_optimized.GPTOptim"
   },
   "block_list": [
-    5148841,
-    5148846,
-    5148850,
-    5148855,
-    5148860,
-    5148864,
-    5148869,
-    5148874,
-    5148878,
-    5148883
+    5148917,
+    5148921,
+    5148925,
+    5148929,
+    5148933,
+    5148938,
+    5148942,
+    5148946,
+    5148951,
+    5148955
   ],
   "block_size": 1024,
   "bos_token_id": 50256,
   "embd_pdrop": 0.1,
   "eos_token_id": 50256,
   "initializer_range": 0.02,
-  "inner_step": 5,
+  "inner_step": 15,
   "inner_steps": 0,
   "last_allreduce_block": 5148769,
   "layer_norm_epsilon": 1e-05,

inner_optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:71e4f0a3c3d015ccbe172d2987a3b997b207ec0448344c8b55f409b6b561fd3a
+oid sha256:dca660e596251a51ce60e31c47a5c6f6f45899815731ff430a2b2c154be13b83
 size 8081781770

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa03d6295f252acd5ca6e19237074da4c08ad514399b39acbfe964b61e6a0545
+oid sha256:4c2c0ca6c9528adc4bd21f46d5031b35cdf015a2b5fc819181afeebc1a1bf76a
 size 4040701744