Add config.json

config.json

@@ -0,0 +1,33 @@
+{  
+  "architectures": ["MoEPForCausalLM"],
+  "auto_map": {
+    "AutoConfig": "moep.MoEPLMConfig",
+    "AutoModelForCausalLM": "moep.MoEPForCausalLM"
+  }
+  ,
+  "attn_pdrop": 0.1,
+  "bos_token_id": 1,
+  "dim": 384,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 2,
+  "ffn_multp": 4,
+  "layer_norm_epsilon": 1e-05,
+  "mask_token_id": 4,
+  "max_position_embeddings": 1024,
+  "model_type": "moep_lm",
+  "moe_k": 2,
+  "n_experts": 4,
+  "n_heads": 6,
+  "n_layers": 12,
+  "n_parallel_blocks": 4,
+  "pad_token_id": 3,
+  "parallel_dim": 192,
+  "parallel_k": 2,
+  "parallel_n_heads": 3,
+  "resid_pdrop": 0.1,
+  "sliding_window": null,
+  "torch_dtype": "float32",
+  "transformers_version": "4.51.3",
+  "vocab_size": 16384,
+  "weight_tyingtrue": true
+}