| { | |
| "architectures": [ | |
| "MaccyForCausalLM" | |
| ], | |
| "auto_map": { | |
| "AutoConfig": "configuration_maccy.MaccyConfig", | |
| "AutoModelForCausalLM": "modeling_maccy.MaccyForCausalLM" | |
| }, | |
| "bias": false, | |
| "bos_token_id": 32759, | |
| "channel_mixer_pattern": "moe", | |
| "context_length": 1024, | |
| "d_model": 576, | |
| "dtype": "float32", | |
| "eos_token_id": 32763, | |
| "mixer_pattern": "kda,kda,kda,mla", | |
| "mla": { | |
| "content_head_dim": 64, | |
| "gated": false, | |
| "kv_rank": 72, | |
| "query_rank": 144, | |
| "rope_head_dim": 32, | |
| "value_head_dim": 64 | |
| }, | |
| "mlp_expansion": 3, | |
| "model_type": "maccy", | |
| "moe": { | |
| "capacity_factor": 1.0, | |
| "expert_expansion": 1.5, | |
| "experts_per_token": 2, | |
| "load_balancing_weight": 0.01, | |
| "n_experts": 4 | |
| }, | |
| "n_heads": 9, | |
| "n_layers": 12, | |
| "pad_token_id": 32759, | |
| "tie_word_embeddings": true, | |
| "transformers_version": "5.14.1", | |
| "use_cache": false, | |
| "vocab_size": 32768 | |
| } | |