ProMoE-L-256/transformer/config.json

{
  "_class_name": "ProMoETransformer2DModel",
  "architecture": "promoe_tc",
  "model_config": {
    "MoE_config": {
      "init_MoeMLP": false,
      "interleave": true,
      "moe_intermediate_size": 2048,
      "num_routed_experts": 12,
      "shared_expert_intermediate_size": 2048,
      "top_k": 1,
      "use_shared_expert": true,
      "use_uncond_expert": true
    },
    "depth": 24,
    "hidden_size": 1024,
    "input_size": 32,
    "num_classes": 1000,
    "num_heads": 16,
    "patch_size": 2
  }
}