Upload folder using huggingface_hub

config.json

@@ -115,54 +115,7 @@
             }
         },
         "ignore": [
-            "lm_head",
-            "model.layers.0.mlp.gate_proj",
-            "model.layers.1.mlp.shared_experts.gate_proj",
-            "model.layers.10.mlp.shared_experts.gate_proj",
-            "model.layers.11.mlp.shared_experts.gate_proj",
-            "model.layers.12.mlp.shared_experts.gate_proj",
-            "model.layers.13.mlp.shared_experts.gate_proj",
-            "model.layers.14.mlp.shared_experts.gate_proj",
-            "model.layers.15.mlp.shared_experts.gate_proj",
-            "model.layers.16.mlp.shared_experts.gate_proj",
-            "model.layers.17.mlp.shared_experts.gate_proj",
-            "model.layers.18.mlp.shared_experts.gate_proj",
-            "model.layers.19.mlp.shared_experts.gate_proj",
-            "model.layers.2.mlp.shared_experts.gate_proj",
-            "model.layers.20.mlp.shared_experts.gate_proj",
-            "model.layers.21.mlp.shared_experts.gate_proj",
-            "model.layers.22.mlp.shared_experts.gate_proj",
-            "model.layers.23.mlp.shared_experts.gate_proj",
-            "model.layers.24.mlp.shared_experts.gate_proj",
-            "model.layers.25.mlp.shared_experts.gate_proj",
-            "model.layers.26.mlp.shared_experts.gate_proj",
-            "model.layers.27.mlp.shared_experts.gate_proj",
-            "model.layers.28.mlp.shared_experts.gate_proj",
-            "model.layers.29.mlp.shared_experts.gate_proj",
-            "model.layers.3.mlp.shared_experts.gate_proj",
-            "model.layers.30.mlp.shared_experts.gate_proj",
-            "model.layers.31.mlp.shared_experts.gate_proj",
-            "model.layers.32.mlp.shared_experts.gate_proj",
-            "model.layers.33.mlp.shared_experts.gate_proj",
-            "model.layers.34.mlp.shared_experts.gate_proj",
-            "model.layers.35.mlp.shared_experts.gate_proj",
-            "model.layers.36.mlp.shared_experts.gate_proj",
-            "model.layers.37.mlp.shared_experts.gate_proj",
-            "model.layers.38.mlp.shared_experts.gate_proj",
-            "model.layers.39.mlp.shared_experts.gate_proj",
-            "model.layers.4.mlp.shared_experts.gate_proj",
-            "model.layers.40.mlp.shared_experts.gate_proj",
-            "model.layers.41.mlp.shared_experts.gate_proj",
-            "model.layers.42.mlp.shared_experts.gate_proj",
-            "model.layers.43.mlp.shared_experts.gate_proj",
-            "model.layers.44.mlp.shared_experts.gate_proj",
-            "model.layers.45.mlp.shared_experts.gate_proj",
-            "model.layers.46.mlp.shared_experts.gate_proj",
-            "model.layers.5.mlp.shared_experts.gate_proj",
-            "model.layers.6.mlp.shared_experts.gate_proj",
-            "model.layers.7.mlp.shared_experts.gate_proj",
-            "model.layers.8.mlp.shared_experts.gate_proj",
-            "model.layers.9.mlp.shared_experts.gate_proj"
+            "lm_head"
         ],
         "quant_algo": "W8A16",
         "producer": {

hf_quant_config.json

@@ -7,54 +7,7 @@
         "quant_algo": "W8A16",
         "kv_cache_quant_algo": null,
         "exclude_modules": [
-            "lm_head",
-            "model.layers.0.mlp.gate_proj",
-            "model.layers.1.mlp.shared_experts.gate_proj",
-            "model.layers.10.mlp.shared_experts.gate_proj",
-            "model.layers.11.mlp.shared_experts.gate_proj",
-            "model.layers.12.mlp.shared_experts.gate_proj",
-            "model.layers.13.mlp.shared_experts.gate_proj",
-            "model.layers.14.mlp.shared_experts.gate_proj",
-            "model.layers.15.mlp.shared_experts.gate_proj",
-            "model.layers.16.mlp.shared_experts.gate_proj",
-            "model.layers.17.mlp.shared_experts.gate_proj",
-            "model.layers.18.mlp.shared_experts.gate_proj",
-            "model.layers.19.mlp.shared_experts.gate_proj",
-            "model.layers.2.mlp.shared_experts.gate_proj",
-            "model.layers.20.mlp.shared_experts.gate_proj",
-            "model.layers.21.mlp.shared_experts.gate_proj",
-            "model.layers.22.mlp.shared_experts.gate_proj",
-            "model.layers.23.mlp.shared_experts.gate_proj",
-            "model.layers.24.mlp.shared_experts.gate_proj",
-            "model.layers.25.mlp.shared_experts.gate_proj",
-            "model.layers.26.mlp.shared_experts.gate_proj",
-            "model.layers.27.mlp.shared_experts.gate_proj",
-            "model.layers.28.mlp.shared_experts.gate_proj",
-            "model.layers.29.mlp.shared_experts.gate_proj",
-            "model.layers.3.mlp.shared_experts.gate_proj",
-            "model.layers.30.mlp.shared_experts.gate_proj",
-            "model.layers.31.mlp.shared_experts.gate_proj",
-            "model.layers.32.mlp.shared_experts.gate_proj",
-            "model.layers.33.mlp.shared_experts.gate_proj",
-            "model.layers.34.mlp.shared_experts.gate_proj",
-            "model.layers.35.mlp.shared_experts.gate_proj",
-            "model.layers.36.mlp.shared_experts.gate_proj",
-            "model.layers.37.mlp.shared_experts.gate_proj",
-            "model.layers.38.mlp.shared_experts.gate_proj",
-            "model.layers.39.mlp.shared_experts.gate_proj",
-            "model.layers.4.mlp.shared_experts.gate_proj",
-            "model.layers.40.mlp.shared_experts.gate_proj",
-            "model.layers.41.mlp.shared_experts.gate_proj",
-            "model.layers.42.mlp.shared_experts.gate_proj",
-            "model.layers.43.mlp.shared_experts.gate_proj",
-            "model.layers.44.mlp.shared_experts.gate_proj",
-            "model.layers.45.mlp.shared_experts.gate_proj",
-            "model.layers.46.mlp.shared_experts.gate_proj",
-            "model.layers.5.mlp.shared_experts.gate_proj",
-            "model.layers.6.mlp.shared_experts.gate_proj",
-            "model.layers.7.mlp.shared_experts.gate_proj",
-            "model.layers.8.mlp.shared_experts.gate_proj",
-            "model.layers.9.mlp.shared_experts.gate_proj"
+            "lm_head"
         ]
     }
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b89f4089f79e8ed75ffab5502f411e0f163ec1d4dc31ab173b8490bd9473ca31
-size 49839252352
+oid sha256:d16fe9402a1d9d91bd042840e31cfa7dd9f7d13e685824387c7bf1d5a2d3001b
+size 49695882864

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84dad1ccfe088bca7359a69f5d4833b30b3633b9db383da11cfeb074ae011f63
-size 8698788944
+oid sha256:385786ac66776950162d4e7b1da147f06ec68dfd297c1511a65b17d5a5812d3e
+size 8676812696

model.safetensors.index.json

@@ -1,7 +1,7 @@
 {
   "metadata": {
     "total_parameters": 29943390976,
-    "total_size": 58536807424
+    "total_size": 58371456000
   },
   "weight_map": {
     "lm_head.weight": "model-00001-of-00002.safetensors",
@@ -10,6 +10,7 @@
     "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -223,6 +224,7 @@
     "model.layers.1.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -436,6 +438,7 @@
     "model.layers.10.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -649,6 +652,7 @@
     "model.layers.11.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -862,6 +866,7 @@
     "model.layers.12.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -1075,6 +1080,7 @@
     "model.layers.13.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -1288,6 +1294,7 @@
     "model.layers.14.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -1501,6 +1508,7 @@
     "model.layers.15.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -1714,6 +1722,7 @@
     "model.layers.16.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -1927,6 +1936,7 @@
     "model.layers.17.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -2140,6 +2150,7 @@
     "model.layers.18.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -2353,6 +2364,7 @@
     "model.layers.19.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -2566,6 +2578,7 @@
     "model.layers.2.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -2779,6 +2792,7 @@
     "model.layers.20.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -2992,6 +3006,7 @@
     "model.layers.21.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -3205,6 +3220,7 @@
     "model.layers.22.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -3418,6 +3434,7 @@
     "model.layers.23.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -3631,6 +3648,7 @@
     "model.layers.24.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.24.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.24.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.24.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -3844,6 +3862,7 @@
     "model.layers.25.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.25.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.25.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.25.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.25.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.25.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -4057,6 +4076,7 @@
     "model.layers.26.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.26.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.26.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.26.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.26.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.26.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -4270,6 +4290,7 @@
     "model.layers.27.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.27.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.27.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.27.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.27.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.27.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -4483,6 +4504,7 @@
     "model.layers.28.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.28.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.28.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.28.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.28.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.28.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -4696,6 +4718,7 @@
     "model.layers.29.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.29.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.29.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.29.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.29.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.29.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.29.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -4909,6 +4932,7 @@
     "model.layers.3.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -5122,6 +5146,7 @@
     "model.layers.30.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.30.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.30.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.30.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.30.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.30.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -5335,6 +5360,7 @@
     "model.layers.31.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.31.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.31.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.31.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.31.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.31.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.31.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -5548,6 +5574,7 @@
     "model.layers.32.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.32.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.32.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.32.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.32.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.32.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.32.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -5761,6 +5788,7 @@
     "model.layers.33.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.33.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.33.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.33.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.33.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.33.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.33.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -5974,6 +6002,7 @@
     "model.layers.34.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.34.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.34.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.34.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.34.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.34.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.34.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -6187,6 +6216,7 @@
     "model.layers.35.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.35.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.35.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.35.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.35.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.35.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.35.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -6400,6 +6430,7 @@
     "model.layers.36.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.36.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.36.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.36.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.36.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.36.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.36.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -6613,6 +6644,7 @@
     "model.layers.37.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.37.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.37.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.37.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.37.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.37.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.37.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -6826,6 +6858,7 @@
     "model.layers.38.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.38.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.38.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.38.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.38.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.38.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.38.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -7039,6 +7072,7 @@
     "model.layers.39.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.39.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.39.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.39.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.39.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.39.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.39.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -7252,6 +7286,7 @@
     "model.layers.4.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -7465,6 +7500,7 @@
     "model.layers.40.mlp.shared_experts.down_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.40.mlp.shared_experts.down_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.40.mlp.shared_experts.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.40.mlp.shared_experts.gate_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.40.mlp.shared_experts.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.40.mlp.shared_experts.up_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.40.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
@@ -7678,6 +7714,7 @@
     "model.layers.41.mlp.shared_experts.down_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.41.mlp.shared_experts.down_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.41.mlp.shared_experts.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.41.mlp.shared_experts.gate_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.41.mlp.shared_experts.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.41.mlp.shared_experts.up_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.41.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
@@ -7891,6 +7928,7 @@
     "model.layers.42.mlp.shared_experts.down_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.42.mlp.shared_experts.down_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.42.mlp.shared_experts.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.42.mlp.shared_experts.gate_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.42.mlp.shared_experts.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.42.mlp.shared_experts.up_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.42.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
@@ -8104,6 +8142,7 @@
     "model.layers.43.mlp.shared_experts.down_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.43.mlp.shared_experts.down_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.43.mlp.shared_experts.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.43.mlp.shared_experts.gate_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.43.mlp.shared_experts.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.43.mlp.shared_experts.up_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.43.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
@@ -8317,6 +8356,7 @@
     "model.layers.44.mlp.shared_experts.down_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.44.mlp.shared_experts.down_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.44.mlp.shared_experts.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.44.mlp.shared_experts.gate_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.44.mlp.shared_experts.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.44.mlp.shared_experts.up_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.44.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
@@ -8530,6 +8570,7 @@
     "model.layers.45.mlp.shared_experts.down_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.45.mlp.shared_experts.down_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.45.mlp.shared_experts.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.45.mlp.shared_experts.gate_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.45.mlp.shared_experts.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.45.mlp.shared_experts.up_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.45.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
@@ -8743,6 +8784,7 @@
     "model.layers.46.mlp.shared_experts.down_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.46.mlp.shared_experts.down_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.46.mlp.shared_experts.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.46.mlp.shared_experts.gate_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.46.mlp.shared_experts.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.46.mlp.shared_experts.up_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.46.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
@@ -8956,6 +8998,7 @@
     "model.layers.5.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -9169,6 +9212,7 @@
     "model.layers.6.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -9382,6 +9426,7 @@
     "model.layers.7.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -9595,6 +9640,7 @@
     "model.layers.8.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
@@ -9808,6 +9854,7 @@
     "model.layers.9.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.shared_experts.down_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.shared_experts.gate_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.shared_experts.up_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",