Checkpoint step 2000, loss=5.3471

config.json

@@ -6,6 +6,7 @@
   "attention_dropout": 0.0,
   "attention_multiplier": 0.015625,
   "bos_token_id": 100257,
+  "dtype": "bfloat16",
   "embedding_multiplier": 12,
   "eos_token_id": 100257,
   "hidden_act": "silu",
@@ -71,8 +72,7 @@
   "router_aux_loss_coef": 0.01,
   "shared_intermediate_size": 2048,
   "tie_word_embeddings": true,
-  "torch_dtype": "float32",
-  "transformers_version": "4.53.3",
+  "transformers_version": "4.57.3",
   "use_cache": false,
   "vocab_size": 116481
 }

generation_config.json

@@ -3,5 +3,5 @@
   "bos_token_id": 100257,
   "eos_token_id": 100257,
   "pad_token_id": 100256,
-  "transformers_version": "4.53.3"
+  "transformers_version": "4.57.3"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a592dc5e5dad07bec7f11fe680d89ffd7e109dd408e264f40aa0e248d4c96193
-size 1475610400
+oid sha256:524bc93bc87da1cb98290fdd46bc975479b705726b8d837151be5007a06850ed
+size 737818416