Upload 7 files

config.json

@@ -3,31 +3,55 @@
     "Lfm2ForCausalLM"
   ],
   "block_auto_adjust_ff_dim": true,
+  "block_dim": 1024,
+  "block_ff_dim": 6656,
   "block_ffn_dim_multiplier": 1.0,
+  "block_mlp_init_scale": 1.0,
   "block_multiple_of": 8,
+  "block_norm_eps": 1e-05,
+  "block_out_init_scale": 1.0,
+  "block_use_swiglu": true,
+  "block_use_xavier_init": true,
   "bos_token_id": 1,
   "conv_L_cache": 3,
   "conv_bias": false,
-  "eos_token_id": 2,
+  "conv_dim": 1024,
+  "conv_dim_out": 1024,
+  "conv_use_xavier_init": true,
+  "eos_token_id": 7,
   "hidden_size": 16,
   "initializer_range": 0.02,
   "intermediate_size": 12,
   "layer_types": [
+    "conv",
+    "conv",
     "full_attention",
+    "conv",
+    "conv",
     "full_attention",
+    "conv",
+    "conv",
     "full_attention",
-    "full_attention"
+    "conv",
+    "full_attention",
+    "conv",
+    "full_attention",
+    "conv",
+    "full_attention",
+    "conv"
   ],
   "max_position_embeddings": 128000,
   "model_type": "lfm2",
   "norm_eps": 1e-05,
   "num_attention_heads": 4,
+  "num_heads": 16,
   "num_hidden_layers": 4,
   "num_key_value_heads": 4,
   "pad_token_id": 0,
   "rope_theta": 1000000.0,
-  "torch_dtype": "float32",
-  "transformers_version": "4.55.4",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.54.0",
   "use_cache": true,
+  "use_pos_enc": true,
   "vocab_size": 65536
 }

generation_config.json

@@ -1,7 +1,7 @@
 {
   "_from_model_config": true,
   "bos_token_id": 1,
-  "eos_token_id": 2,
+  "eos_token_id": 7,
   "pad_token_id": 0,
-  "transformers_version": "4.55.4"
+  "transformers_version": "4.54.0"
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:396d9f677e05a8840053d4773a3db96231ad60d674dd7f4d097d987c1e532b03
-size 4232657
+oid sha256:d75e7bc19ebd90d1d0ca09566231788bdb098789b5f03ac59725ca28d0eb995e
+size 2120950