Outer Step 1. Inner Step 390. Batch Size 320

config.json

@@ -268,23 +268,23 @@
     "AutoModelForCausalLM": "distributed/optimized-gpt2-500m--modeling_gpt_optimized.GPTOptim"
   },
   "block_list": [
-    5146604,
-    5146608,
-    5146612,
-    5146616,
-    5146619,
-    5146623,
-    5146627,
-    5146631,
-    5146635,
-    5146639
+    5146671,
+    5146675,
+    5146679,
+    5146683,
+    5146687,
+    5146691,
+    5146695,
+    5146698,
+    5146703,
+    5146707
   ],
   "block_size": 1024,
   "bos_token_id": 50256,
   "embd_pdrop": 0.1,
   "eos_token_id": 50256,
   "initializer_range": 0.02,
-  "inner_step": 380,
+  "inner_step": 390,
   "inner_steps": 0,
   "last_allreduce_block": 5143202,
   "layer_norm_epsilon": 1e-05,

inner_optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:23075319b90ebaaa4c0990900a82795b462f2e431d05cb9d0316046ea0af8ff0
+oid sha256:a316e358f22db16503474f9b59485daa8b50f1eb8990cf8e0ad91d6d7c5b11fb
 size 8081781770

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a302705d75b0569444e5eca6634c8367162acbde66cd439559349853405ef98
+oid sha256:ad9873b70289735f467d6c563f40575bdfde34c6424344d1048ddc5cc13e031b
 size 4040701744