{ "source_model": "deepseek-ai/DeepSeek-V4-Flash", "extraction_timestamp": "2026-07-30T16:07:42Z", "dtype": "bfloat16", "total_experts": 260, "layers": { "0": { "expert_ids": [ 1, 2, 3, 4, 5 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "1": { "expert_ids": [ 1, 2, 3 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "2": { "expert_ids": [ 1, 2, 3, 4 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "3": { "expert_ids": [ 0, 1, 7 ], "experts": { "0": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "4": { "expert_ids": [ 0, 1, 5 ], "experts": { "0": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "5": { "expert_ids": [ 1, 2, 3, 4, 5, 6, 7 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "6": { "expert_ids": [ 0, 1, 7 ], "experts": { "0": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "7": { "expert_ids": [ 1, 2, 3, 5, 6, 9 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "9": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "8": { "expert_ids": [ 3, 4, 5, 6, 7, 8, 10 ], "experts": { "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "8": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "10": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "10": { "expert_ids": [ 1, 3, 6, 8 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "8": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "11": { "expert_ids": [ 1, 3, 4, 6, 7, 8 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "8": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "12": { "expert_ids": [ 1, 5, 7 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "13": { "expert_ids": [ 1, 2, 3, 4, 5, 6 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "14": { "expert_ids": [ 1, 2, 3, 4, 5, 6, 7 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "15": { "expert_ids": [ 0, 1, 2, 3, 4, 5, 6, 7 ], "experts": { "0": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "16": { "expert_ids": [ 0, 1, 4, 5, 8 ], "experts": { "0": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "8": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "17": { "expert_ids": [ 0, 1, 2, 3, 4, 6 ], "experts": { "0": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "18": { "expert_ids": [ 0, 1, 2, 3, 4, 6 ], "experts": { "0": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "19": { "expert_ids": [ 1, 3, 5, 6, 7, 8, 10 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "8": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "10": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "20": { "expert_ids": [ 4, 7, 8 ], "experts": { "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "8": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "21": { "expert_ids": [ 2, 3 ], "experts": { "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "22": { "expert_ids": [ 1, 2, 3, 4, 6, 7, 8, 9, 10 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "8": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "9": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "10": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "23": { "expert_ids": [ 3, 5, 6, 7, 9, 10, 14 ], "experts": { "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "9": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "10": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "14": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "24": { "expert_ids": [ 2, 3, 4, 12 ], "experts": { "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "12": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "25": { "expert_ids": [ 2, 3, 4, 6, 7, 8, 9, 13, 14, 19, 23 ], "experts": { "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "8": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "9": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "13": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "14": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "19": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "23": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "26": { "expert_ids": [ 9 ], "experts": { "9": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "27": { "expert_ids": [ 1, 2, 3, 4, 5, 14, 28 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "14": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "28": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "28": { "expert_ids": [ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15 ], "experts": { "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "8": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "9": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "10": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "11": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "14": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "15": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "29": { "expert_ids": [ 1, 3, 4, 9, 10, 13, 15 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "9": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "10": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "13": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "15": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "30": { "expert_ids": [ 13, 14 ], "experts": { "13": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "14": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "31": { "expert_ids": [ 3, 6, 12, 13, 14, 18 ], "experts": { "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "12": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "13": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "14": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "18": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "32": { "expert_ids": [ 1, 4, 5, 6, 10, 13, 15, 21 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "10": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "13": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "15": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "21": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "33": { "expert_ids": [ 3, 12, 13, 17 ], "experts": { "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "12": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "13": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "17": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "34": { "expert_ids": [ 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 21 ], "experts": { "1": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "8": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "9": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "10": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "11": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "12": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "13": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "15": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "21": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "35": { "expert_ids": [ 2, 3, 4, 6, 7, 8, 11, 15 ], "experts": { "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "8": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "11": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "15": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "36": { "expert_ids": [ 2, 3, 5, 7, 9, 10, 11, 12, 14, 15, 16, 19 ], "experts": { "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "9": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "10": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "11": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "12": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "14": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "15": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "16": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "19": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "37": { "expert_ids": [ 2, 3, 5, 6, 7, 10, 12 ], "experts": { "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "10": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "12": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "38": { "expert_ids": [ 2, 3, 4, 6, 7, 9, 10, 11, 12, 15 ], "experts": { "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "9": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "10": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "11": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "12": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "15": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "39": { "expert_ids": [ 2, 3, 4, 7, 10, 13, 16, 20 ], "experts": { "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "10": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "13": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "16": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "20": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "40": { "expert_ids": [ 3, 13 ], "experts": { "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "13": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "41": { "expert_ids": [ 3, 4, 5, 7, 8, 9, 10, 12, 14 ], "experts": { "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "8": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "9": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "10": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "12": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "14": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } }, "42": { "expert_ids": [ 2, 3, 4, 5, 6, 7, 11, 12 ], "experts": { "2": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "3": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "4": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "5": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "6": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "7": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "11": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 }, "12": { "gate_proj": [ 2048, 4096 ], "up_proj": [ 2048, 4096 ], "down_proj": [ 4096, 2048 ], "params": 25165824 } } } }, "shard_files": [ "coding-experts-00000.safetensors", "coding-experts-00001.safetensors" ], "total_parameters": 6543114240 }