{
  "architectures": [
    "MaccyForCausalLM"
  ],
  "auto_map": {
    "AutoConfig": "configuration_maccy.MaccyConfig",
    "AutoModelForCausalLM": "modeling_maccy.MaccyForCausalLM"
  },
  "bias": false,
  "bos_token_id": 32759,
  "channel_mixer_pattern": "moe",
  "context_length": 1024,
  "d_model": 576,
  "dtype": "float32",
  "eos_token_id": 32763,
  "mixer_pattern": "kda,kda,kda,mla",
  "mla": {
    "content_head_dim": 64,
    "gated": false,
    "kv_rank": 72,
    "query_rank": 144,
    "rope_head_dim": 32,
    "value_head_dim": 64
  },
  "mlp_expansion": 3,
  "model_type": "maccy",
  "moe": {
    "capacity_factor": 1.0,
    "expert_expansion": 1.5,
    "experts_per_token": 2,
    "load_balancing_weight": 0.01,
    "n_experts": 4
  },
  "n_heads": 9,
  "n_layers": 12,
  "pad_token_id": 32759,
  "tie_word_embeddings": true,
  "transformers_version": "5.14.1",
  "use_cache": false,
  "vocab_size": 32768
}