Outer Step 4. Inner Step 0. Batch Size 0

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "thieu86/SN3801",
+  "_name_or_path": "distributed/optimized-gpt2-1b",
   "activation_function": "gelu_new",
   "all_reduce_scores": {
     "0": "NON_PARTICIPATING",
@@ -16,7 +16,7 @@
     "108": "NON_PARTICIPATING",
     "109": "NON_PARTICIPATING",
     "11": "NON_PARTICIPATING",
-    "110": "SUCCESS",
+    "110": "NON_PARTICIPATING",
     "111": "NON_PARTICIPATING",
     "112": "NON_PARTICIPATING",
     "113": "NON_PARTICIPATING",
@@ -33,14 +33,14 @@
     "123": "NON_PARTICIPATING",
     "124": "NON_PARTICIPATING",
     "125": "NON_PARTICIPATING",
-    "126": "SUCCESS",
-    "127": "SUCCESS",
+    "126": "NON_PARTICIPATING",
+    "127": "NON_PARTICIPATING",
     "128": "NON_PARTICIPATING",
     "129": "NON_PARTICIPATING",
     "13": "NON_PARTICIPATING",
     "130": "NON_PARTICIPATING",
     "131": "NON_PARTICIPATING",
-    "132": "SUCCESS",
+    "132": "NON_PARTICIPATING",
     "133": "NON_PARTICIPATING",
     "134": "NON_PARTICIPATING",
     "135": "NON_PARTICIPATING",
@@ -71,7 +71,7 @@
     "158": "NON_PARTICIPATING",
     "159": "NON_PARTICIPATING",
     "16": "NON_PARTICIPATING",
-    "160": "SUCCESS",
+    "160": "NON_PARTICIPATING",
     "161": "NON_PARTICIPATING",
     "162": "NON_PARTICIPATING",
     "163": "NON_PARTICIPATING",
@@ -82,12 +82,12 @@
     "168": "NON_PARTICIPATING",
     "169": "NON_PARTICIPATING",
     "17": "NON_PARTICIPATING",
-    "170": "SUCCESS",
+    "170": "NON_PARTICIPATING",
     "171": "NON_PARTICIPATING",
-    "172": "SUCCESS",
+    "172": "NON_PARTICIPATING",
     "173": "NON_PARTICIPATING",
     "174": "NON_PARTICIPATING",
-    "175": "SUCCESS",
+    "175": "NON_PARTICIPATING",
     "176": "NON_PARTICIPATING",
     "177": "NON_PARTICIPATING",
     "178": "NON_PARTICIPATING",
@@ -99,7 +99,7 @@
     "183": "NON_PARTICIPATING",
     "184": "NON_PARTICIPATING",
     "185": "NON_PARTICIPATING",
-    "186": "SUCCESS",
+    "186": "NON_PARTICIPATING",
     "187": "NON_PARTICIPATING",
     "188": "NON_PARTICIPATING",
     "189": "NON_PARTICIPATING",
@@ -144,7 +144,7 @@
     "223": "NON_PARTICIPATING",
     "224": "NON_PARTICIPATING",
     "225": "NON_PARTICIPATING",
-    "226": "SUCCESS",
+    "226": "NON_PARTICIPATING",
     "227": "NON_PARTICIPATING",
     "228": "NON_PARTICIPATING",
     "229": "NON_PARTICIPATING",
@@ -171,7 +171,7 @@
     "248": "NON_PARTICIPATING",
     "249": "SUCCESS",
     "25": "SUCCESS",
-    "250": "SUCCESS",
+    "250": "NON_PARTICIPATING",
     "251": "NON_PARTICIPATING",
     "252": "NON_PARTICIPATING",
     "253": "NON_PARTICIPATING",
@@ -193,14 +193,14 @@
     "38": "NON_PARTICIPATING",
     "39": "NON_PARTICIPATING",
     "4": "NON_PARTICIPATING",
-    "40": "SUCCESS",
+    "40": "NON_PARTICIPATING",
     "41": "NON_PARTICIPATING",
     "42": "NON_PARTICIPATING",
     "43": "NON_PARTICIPATING",
     "44": "NON_PARTICIPATING",
     "45": "NON_PARTICIPATING",
     "46": "NON_PARTICIPATING",
-    "47": "SUCCESS",
+    "47": "NON_PARTICIPATING",
     "48": "NON_PARTICIPATING",
     "49": "NON_PARTICIPATING",
     "5": "NON_PARTICIPATING",
@@ -210,7 +210,7 @@
     "53": "NON_PARTICIPATING",
     "54": "NON_PARTICIPATING",
     "55": "NON_PARTICIPATING",
-    "56": "SUCCESS",
+    "56": "NON_PARTICIPATING",
     "57": "NON_PARTICIPATING",
     "58": "NON_PARTICIPATING",
     "59": "NON_PARTICIPATING",
@@ -223,7 +223,7 @@
     "65": "NON_PARTICIPATING",
     "66": "NON_PARTICIPATING",
     "67": "SUCCESS",
-    "68": "SUCCESS",
+    "68": "NON_PARTICIPATING",
     "69": "NON_PARTICIPATING",
     "7": "NON_PARTICIPATING",
     "70": "NON_PARTICIPATING",
@@ -231,18 +231,18 @@
     "72": "NON_PARTICIPATING",
     "73": "NON_PARTICIPATING",
     "74": "NON_PARTICIPATING",
-    "75": "SUCCESS",
+    "75": "NON_PARTICIPATING",
     "76": "NON_PARTICIPATING",
     "77": "NON_PARTICIPATING",
     "78": "NON_PARTICIPATING",
     "79": "NON_PARTICIPATING",
     "8": "NON_PARTICIPATING",
     "80": "NON_PARTICIPATING",
-    "81": "SUCCESS",
-    "82": "SUCCESS",
+    "81": "NON_PARTICIPATING",
+    "82": "NON_PARTICIPATING",
     "83": "NON_PARTICIPATING",
-    "84": "SUCCESS",
-    "85": "SUCCESS",
+    "84": "NON_PARTICIPATING",
+    "85": "NON_PARTICIPATING",
     "86": "NON_PARTICIPATING",
     "87": "NON_PARTICIPATING",
     "88": "NON_PARTICIPATING",
@@ -257,7 +257,7 @@
     "96": "NON_PARTICIPATING",
     "97": "NON_PARTICIPATING",
     "98": "NON_PARTICIPATING",
-    "99": "SUCCESS"
+    "99": "NON_PARTICIPATING"
   },
   "architectures": [
     "GPTOptim"
@@ -267,26 +267,15 @@
     "AutoConfig": "distributed/optimized-gpt2-500m--configuration_gpt_optimized.GPTOptimConfig",
     "AutoModelForCausalLM": "distributed/optimized-gpt2-500m--modeling_gpt_optimized.GPTOptim"
   },
-  "block_list": [
-    5184439,
-    5184443,
-    5184447,
-    5184451,
-    5184455,
-    5184458,
-    5184463,
-    5184467,
-    5184471,
-    5184475
-  ],
+  "block_list": [],
   "block_size": 1024,
   "bos_token_id": 50256,
   "embd_pdrop": 0.1,
   "eos_token_id": 50256,
   "initializer_range": 0.02,
-  "inner_step": 1648,
+  "inner_step": 0,
   "inner_steps": 0,
-  "last_allreduce_block": 5160840,
+  "last_allreduce_block": 5269293,
   "layer_norm_epsilon": 1e-05,
   "model_type": "gpt_optimized",
   "n_embd": 1280,

inner_optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:587b7527aa6a015c809122bd06cd16f894156fa78080ae7fc513bd6ceac145e3
-size 8081781770
+oid sha256:d34614510f6061b01344adc2bd6ae24bf6540125cd79e18a74841aa3b38f1ab7
+size 2944

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69e913340305089feac21f70382cefbcae7bbe647b3f22ceb69884f830629bc6
+oid sha256:4c7592a774622f4525d900fb81f89c54dc01fe659c7e82d317ce5527c17bd007
 size 4040701744