Training in progress, step 1000

config.json

@@ -8,7 +8,7 @@
   "dtype": "float32",
   "embedding_size": 128,
   "expert_intermediate_size": 2624,
-  "group_depth": 3,
+  "group_depth": 2,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
@@ -23,7 +23,7 @@
   "num_attention_heads": 12,
   "num_expert_modules": 2,
   "num_experts": 4,
-  "num_hidden_layers": 15,
+  "num_hidden_layers": 14,
   "pad_token_id": 0,
   "router_jitter_noise": 0.01,
   "top_k": 1,

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d3586fd5ccefec72367fff5e86895474177c0a520f2c5a372e891b4015f04fa
-size 381944306
+oid sha256:62842584bcf2dda53258e996def0ee2a7e1915ba26342af13ff9382f86683cf4
+size 402029570

runs/Oct20_00-04-39_nid007648/events.out.tfevents.1760911496.nid007648.68035.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c031705cab06fa98bc267d086ec0032b90e6e655837417a57099b7c8311d819a
+size 15820

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f1c2b5c6584fb21be0c50e23c85bf35c3dafa694c9050bc06eb22d89190f615
+oid sha256:2dccca07d27100677936cb8a7294295fefbf7e8d8b5b07969e20edc9f40ec97a
 size 5432