Upload MistralForCausalLM

config.json

@@ -7,14 +7,14 @@
   "bos_token_id": 1,
   "eos_token_id": 32000,
   "hidden_act": "silu",
-  "hidden_size": 3072,
+  "hidden_size": 2048,
   "initializer_range": 0.02,
-  "intermediate_size": 5632,
-  "max_position_embeddings": 2048,
+  "intermediate_size": 6144,
+  "max_position_embeddings": 3072,
   "model_type": "mistral",
   "num_attention_heads": 32,
-  "num_hidden_layers": 22,
-  "num_key_value_heads": 4,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 32,
   "pad_token_id": 32000,
   "rms_norm_eps": 1e-05,
   "rope_theta": 10000.0,

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b643df1cf3416ea83b01ade92dab021839dd960f57202d9cc06f20c3b866368
-size 4936818448
+oid sha256:1eba319841c5777f885c39074fea560bad2ef4da703213d31f4308d0cc878f4f
+size 4960655224

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa4ec29a0ca9f34d5d1d53acfb0c7c6e5b2f582f0621bc279beb3a2effead2a9
-size 2287921872
+oid sha256:5d1985d0ebce3c69f343984962f725fa8a78138a5374f40d1dc0f29487f02028
+size 799599248

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 7224717312
+    "total_size": 5760229376
   },
   "weight_map": {
     "lm_head.weight": "model-00002-of-00002.safetensors",
@@ -68,51 +68,51 @@
     "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.18.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
@@ -122,24 +122,42 @@
     "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",