Outer Step 8. Inner Step 112. Batch Size 12

config.json

@@ -268,23 +268,23 @@
     "AutoModelForCausalLM": "distributed/optimized-gpt2-500m--modeling_gpt_optimized.GPTOptim"
   },
   "block_list": [
-    5292001,
-    5292004,
-    5292007,
-    5292011,
-    5292014,
-    5292018,
-    5292021,
-    5292025,
-    5292028,
-    5292032
+    5292075,
+    5292079,
+    5292083,
+    5292086,
+    5292089,
+    5292093,
+    5292097,
+    5292100,
+    5292103,
+    5292107
   ],
   "block_size": 1024,
   "bos_token_id": 50256,
   "embd_pdrop": 0.1,
   "eos_token_id": 50256,
   "initializer_range": 0.02,
-  "inner_step": 100,
+  "inner_step": 112,
   "inner_steps": 0,
   "last_allreduce_block": 5283755,
   "layer_norm_epsilon": 1e-05,

inner_optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f96fbdd99d81f142e6f61304ebd9384b7fde57ad320dcbd375380db28d2a14c7
+oid sha256:29025972e61b4056fec5802d0d89d4e646d23a8a2f17fbc41b4fbd90d10a1824
 size 8081782026

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6fef073a59d72dbb03737d798c75371c6da536957ab27938b617c463d75bae90
+oid sha256:777c4928eb20e4ae93fe99c9a1b4bb681857f56ce22b6c624043dc6ae776a41b
 size 4040701744