Upload 6 files

Model Attn/config_Attn.json

@@ -0,0 +1,21 @@
+{
+  "context_len": 1536,
+  "ffn_dim_multiplier": null,
+  "ffn_hidden_dims": 556,
+  "moe_active_experts": 2,
+  "moe_aux_loss_coef": 0.01,
+  "moe_eps": 1e-06,
+  "moe_num_experts": 2,
+  "moe_shared_experts": 1,
+  "num_dims": 832,
+  "num_heads": 16,
+  "num_kv_heads": 4,
+  "num_layers": 34,
+  "rmsnorm_eps": 1e-06,
+  "rope_theta": 100000.0,
+  "use_cache": false,
+  "use_flash": true,
+  "use_lossfreebalance": false,
+  "use_moe": false,
+  "vocab_size": 49152
+}

Model Attn/model_Attn.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce97dae14abfa2d03873209ec9b3b64bd4e8268fb8459e8cc6ea12ac7f9c3caa
+size 587931736

Model FFN/config_FFN.json

@@ -0,0 +1,21 @@
+{
+  "context_len": 1536,
+  "ffn_dim_multiplier": null,
+  "ffn_hidden_dims": 2048,
+  "moe_active_experts": 2,
+  "moe_aux_loss_coef": 0.01,
+  "moe_eps": 1e-06,
+  "moe_num_experts": 2,
+  "moe_shared_experts": 1,
+  "num_dims": 512,
+  "num_heads": 16,
+  "num_kv_heads": 4,
+  "num_layers": 32,
+  "rmsnorm_eps": 1e-06,
+  "rope_theta": 100000.0,
+  "use_cache": false,
+  "use_flash": true,
+  "use_lossfreebalance": false,
+  "use_moe": false,
+  "vocab_size": 49152
+}

Model FFN/model_FFN.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40d14a8c629b30840dce2f4c19da7e21f120c368647b7bb874f6070bcf31f87e
+size 587364560

Model MoE 2+1/config_moe_2p1_78e3.safetensors.json

@@ -0,0 +1,21 @@
+{
+  "context_len": 1024,
+  "ffn_dim_multiplier": null,
+  "ffn_hidden_dims": 1024,
+  "moe_active_experts": 2,
+  "moe_aux_loss_coef": 0.01,
+  "moe_eps": 1e-06,
+  "moe_num_experts": 2,
+  "moe_shared_experts": 1,
+  "num_dims": 384,
+  "num_heads": 16,
+  "num_kv_heads": 4,
+  "num_layers": 32,
+  "rmsnorm_eps": 1e-06,
+  "rope_theta": 100000.0,
+  "use_cache": false,
+  "use_flash": true,
+  "use_lossfreebalance": true,
+  "use_moe": true,
+  "vocab_size": 49152
+}

Model MoE 2+1/model_moe_2p1_78e3.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4adbb1d15f2f4dd3f88f73a7abcfe32f47a9a74840d985a6536320cc70b8f78e
+size 575923536