Upload 6 files

config.json

@@ -11,19 +11,20 @@
   "initializer_range": 0.02,
   "intermediate_size": 11776,
   "layer_types": [
+    "full_attention",
     "full_attention",
     "conv"
   ],
   "max_position_embeddings": 128000,
   "model_type": "lfm2_moe",
-  "moe_intermediate_size": 1536,
+  "moe_intermediate_size": 16,
   "norm_eps": 1e-05,
   "norm_topk_prob": true,
   "num_attention_heads": 4,
-  "num_dense_layers": 2,
+  "num_dense_layers": 1,
   "num_experts": 64,
   "num_experts_per_tok": 4,
-  "num_hidden_layers": 2,
+  "num_hidden_layers": 3,
   "num_key_value_heads": 4,
   "pad_token_id": 0,
   "rope_parameters": {
@@ -32,7 +33,7 @@
   },
   "routed_scaling_factor": 1.0,
   "tie_word_embeddings": true,
-  "transformers_version": "5.0.0",
+  "transformers_version": "5.1.0",
   "use_cache": true,
   "use_expert_bias": true,
   "vocab_size": 65536

generation_config.json

@@ -5,6 +5,6 @@
   "output_attentions": false,
   "output_hidden_states": false,
   "pad_token_id": 0,
-  "transformers_version": "5.0.0",
+  "transformers_version": "5.1.0",
   "use_cache": true
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d0c40fd1d10795ba70ca3bbf43919fc6a0c7403dc7029a93bed2279fa53c19fe
-size 4364744
+oid sha256:6d766accf534f9d95ba4b64aa30d5ee9b13d7d4f76e04d90b29eeaf5ba2dfdca
+size 3483384