Create config.json

config.json

@@ -0,0 +1,31 @@
+{  
+  "architectures": ["MoEPgemmaForCausalLM"],
+  "auto_map": {
+    "AutoConfig": "moep_gemma.MoEPgemmaLMConfig",
+    "AutoModel": "moep_gemma.MoEPgemmaModel",
+    "AutoModelForCausalLM": "moep_gemma.MoEPgemmaForCausalLM"
+  }
+  ,
+  "bos_token_id": 1,
+  "dim": 384,
+  "eos_token_id": 2,
+  "ffn_multp": 4,
+  "layer_norm_epsilon": 1e-05,
+  "mask_token_id": 4,
+  "max_position_embeddings": 1024,
+  "model_type": "moep_gemma_lm",
+  "moe_k": 2,
+  "n_experts": 4,
+  "n_heads": 6,
+  "n_layers": 12,
+  "n_parallel_blocks": 4,
+  "pad_token_id": 3,
+  "parallel_dim": 192,
+  "parallel_k": 2,
+  "parallel_n_heads": 3,
+  "torch_dtype": "float32",
+  "transformers_version": "4.51.3",
+  "vocab_size": 16384,
+  "weight_tyingtrue": true,
+  "sliding_window": 256
+}