config.json

{  
  "architectures": ["MoEPForCausalLM"],
  "auto_map": {
    "AutoConfig": "moep.MoEPLMConfig",
    "AutoModel": "moep.MoEPModel",
    "AutoModelForCausalLM": "moep.MoEPForCausalLM"
  }
  ,
  "attn_pdrop": 0.1,
  "bos_token_id": 1,
  "dim": 384,
  "embd_pdrop": 0.1,
  "eos_token_id": 2,
  "ffn_multp": 4,
  "layer_norm_epsilon": 1e-05,
  "mask_token_id": 4,
  "max_position_embeddings": 1024,
  "model_type": "moep_lm",
  "moe_k": 2,
  "n_experts": 4,
  "n_heads": 6,
  "n_layers": 12,
  "n_parallel_blocks": 4,
  "pad_token_id": 3,
  "parallel_dim": 192,
  "parallel_k": 2,
  "parallel_n_heads": 3,
  "resid_pdrop": 0.1,
  "sliding_window": null,
  "torch_dtype": "float32",
  "transformers_version": "4.51.3",
  "vocab_size": 16384,
  "weight_tyingtrue": true
}