Hugging Face's logo

Akahsizrr
/
Mini-Whale-Flash

Text Generation
Transformers
Safetensors
code
Mixture of Experts
fusion
deepseek
qwen
mixture-of-experts
research
Model card Files
xet
 Community
Mini-Whale-Flash / coding-experts-v1 /manifest.json
Akahsizrr's picture
Akahsizrr
Upload folder using huggingface_hub
51700b0 verified
Raw
History Blame Contribute Delete
77.2 kB
{
 "source_model": "deepseek-ai/DeepSeek-V4-Flash",
 "extraction_timestamp": "2026-07-30T16:07:42Z",
 "dtype": "bfloat16",
 "total_experts": 260,
 "layers": {
 "0": {
 "expert_ids": [
 1,
 2,
 3,
 4,
 5
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "1": {
 "expert_ids": [
 1,
 2,
 3
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "2": {
 "expert_ids": [
 1,
 2,
 3,
 4
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "3": {
 "expert_ids": [
 0,
 1,
 7
 ],
 "experts": {
 "0": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "4": {
 "expert_ids": [
 0,
 1,
 5
 ],
 "experts": {
 "0": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "5": {
 "expert_ids": [
 1,
 2,
 3,
 4,
 5,
 6,
 7
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "6": {
 "expert_ids": [
 0,
 1,
 7
 ],
 "experts": {
 "0": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "7": {
 "expert_ids": [
 1,
 2,
 3,
 5,
 6,
 9
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "9": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "8": {
 "expert_ids": [
 3,
 4,
 5,
 6,
 7,
 8,
 10
 ],
 "experts": {
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "8": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "10": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "10": {
 "expert_ids": [
 1,
 3,
 6,
 8
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "8": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "11": {
 "expert_ids": [
 1,
 3,
 4,
 6,
 7,
 8
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "8": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "12": {
 "expert_ids": [
 1,
 5,
 7
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "13": {
 "expert_ids": [
 1,
 2,
 3,
 4,
 5,
 6
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "14": {
 "expert_ids": [
 1,
 2,
 3,
 4,
 5,
 6,
 7
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "15": {
 "expert_ids": [
 0,
 1,
 2,
 3,
 4,
 5,
 6,
 7
 ],
 "experts": {
 "0": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "16": {
 "expert_ids": [
 0,
 1,
 4,
 5,
 8
 ],
 "experts": {
 "0": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "8": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "17": {
 "expert_ids": [
 0,
 1,
 2,
 3,
 4,
 6
 ],
 "experts": {
 "0": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "18": {
 "expert_ids": [
 0,
 1,
 2,
 3,
 4,
 6
 ],
 "experts": {
 "0": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "19": {
 "expert_ids": [
 1,
 3,
 5,
 6,
 7,
 8,
 10
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "8": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "10": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "20": {
 "expert_ids": [
 4,
 7,
 8
 ],
 "experts": {
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "8": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "21": {
 "expert_ids": [
 2,
 3
 ],
 "experts": {
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "22": {
 "expert_ids": [
 1,
 2,
 3,
 4,
 6,
 7,
 8,
 9,
 10
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "8": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "9": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "10": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "23": {
 "expert_ids": [
 3,
 5,
 6,
 7,
 9,
 10,
 14
 ],
 "experts": {
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "9": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "10": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "14": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "24": {
 "expert_ids": [
 2,
 3,
 4,
 12
 ],
 "experts": {
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "12": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "25": {
 "expert_ids": [
 2,
 3,
 4,
 6,
 7,
 8,
 9,
 13,
 14,
 19,
 23
 ],
 "experts": {
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "8": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "9": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "13": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "14": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "19": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "23": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "26": {
 "expert_ids": [
 9
 ],
 "experts": {
 "9": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "27": {
 "expert_ids": [
 1,
 2,
 3,
 4,
 5,
 14,
 28
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "14": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "28": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "28": {
 "expert_ids": [
 2,
 3,
 4,
 5,
 6,
 7,
 8,
 9,
 10,
 11,
 14,
 15
 ],
 "experts": {
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "8": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "9": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "10": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "11": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "14": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "15": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "29": {
 "expert_ids": [
 1,
 3,
 4,
 9,
 10,
 13,
 15
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "9": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "10": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "13": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "15": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "30": {
 "expert_ids": [
 13,
 14
 ],
 "experts": {
 "13": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "14": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "31": {
 "expert_ids": [
 3,
 6,
 12,
 13,
 14,
 18
 ],
 "experts": {
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "12": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "13": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "14": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "18": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "32": {
 "expert_ids": [
 1,
 4,
 5,
 6,
 10,
 13,
 15,
 21
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "10": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "13": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "15": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "21": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "33": {
 "expert_ids": [
 3,
 12,
 13,
 17
 ],
 "experts": {
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "12": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "13": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "17": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "34": {
 "expert_ids": [
 1,
 3,
 4,
 5,
 6,
 7,
 8,
 9,
 10,
 11,
 12,
 13,
 15,
 21
 ],
 "experts": {
 "1": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "8": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "9": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "10": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "11": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "12": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "13": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "15": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "21": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "35": {
 "expert_ids": [
 2,
 3,
 4,
 6,
 7,
 8,
 11,
 15
 ],
 "experts": {
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "8": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "11": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "15": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "36": {
 "expert_ids": [
 2,
 3,
 5,
 7,
 9,
 10,
 11,
 12,
 14,
 15,
 16,
 19
 ],
 "experts": {
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "9": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "10": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "11": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "12": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "14": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "15": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "16": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "19": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "37": {
 "expert_ids": [
 2,
 3,
 5,
 6,
 7,
 10,
 12
 ],
 "experts": {
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "10": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "12": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "38": {
 "expert_ids": [
 2,
 3,
 4,
 6,
 7,
 9,
 10,
 11,
 12,
 15
 ],
 "experts": {
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "9": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "10": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "11": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "12": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "15": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "39": {
 "expert_ids": [
 2,
 3,
 4,
 7,
 10,
 13,
 16,
 20
 ],
 "experts": {
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "10": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "13": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "16": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "20": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "40": {
 "expert_ids": [
 3,
 13
 ],
 "experts": {
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "13": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "41": {
 "expert_ids": [
 3,
 4,
 5,
 7,
 8,
 9,
 10,
 12,
 14
 ],
 "experts": {
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "8": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "9": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "10": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "12": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "14": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 },
 "42": {
 "expert_ids": [
 2,
 3,
 4,
 5,
 6,
 7,
 11,
 12
 ],
 "experts": {
 "2": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "3": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "4": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "5": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "6": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "7": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "11": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 },
 "12": {
 "gate_proj": [
 2048,
 4096
 ],
 "up_proj": [
 2048,
 4096
 ],
 "down_proj": [
 4096,
 2048
 ],
 "params": 25165824
 }
 }
 }
 },
 "shard_files": [
 "coding-experts-00000.safetensors",
 "coding-experts-00001.safetensors"
 ],
 "total_parameters": 6543114240
}