| { |
| "model_name": "Qwen/Qwen3.5-35B-A3B-FP8", |
| "num_layers": 40, |
| "d_model": 2048, |
| "vocab_size": 248320, |
| "layers": [ |
| { |
| "index": 0, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.0" |
| }, |
| { |
| "index": 1, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.1" |
| }, |
| { |
| "index": 2, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.2" |
| }, |
| { |
| "index": 3, |
| "attention_type": "standard", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.3" |
| }, |
| { |
| "index": 4, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.4" |
| }, |
| { |
| "index": 5, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.5" |
| }, |
| { |
| "index": 6, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.6" |
| }, |
| { |
| "index": 7, |
| "attention_type": "standard", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.7" |
| }, |
| { |
| "index": 8, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.8" |
| }, |
| { |
| "index": 9, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.9" |
| }, |
| { |
| "index": 10, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.10" |
| }, |
| { |
| "index": 11, |
| "attention_type": "standard", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.11" |
| }, |
| { |
| "index": 12, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.12" |
| }, |
| { |
| "index": 13, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.13" |
| }, |
| { |
| "index": 14, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.14" |
| }, |
| { |
| "index": 15, |
| "attention_type": "standard", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.15" |
| }, |
| { |
| "index": 16, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.16" |
| }, |
| { |
| "index": 17, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.17" |
| }, |
| { |
| "index": 18, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.18" |
| }, |
| { |
| "index": 19, |
| "attention_type": "standard", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.19" |
| }, |
| { |
| "index": 20, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.20" |
| }, |
| { |
| "index": 21, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.21" |
| }, |
| { |
| "index": 22, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.22" |
| }, |
| { |
| "index": 23, |
| "attention_type": "standard", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.23" |
| }, |
| { |
| "index": 24, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.24" |
| }, |
| { |
| "index": 25, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.25" |
| }, |
| { |
| "index": 26, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.26" |
| }, |
| { |
| "index": 27, |
| "attention_type": "standard", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.27" |
| }, |
| { |
| "index": 28, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.28" |
| }, |
| { |
| "index": 29, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.29" |
| }, |
| { |
| "index": 30, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.30" |
| }, |
| { |
| "index": 31, |
| "attention_type": "standard", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.31" |
| }, |
| { |
| "index": 32, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.32" |
| }, |
| { |
| "index": 33, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.33" |
| }, |
| { |
| "index": 34, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.34" |
| }, |
| { |
| "index": 35, |
| "attention_type": "standard", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.35" |
| }, |
| { |
| "index": 36, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.36" |
| }, |
| { |
| "index": 37, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.37" |
| }, |
| { |
| "index": 38, |
| "attention_type": "gated_deltanet", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.38" |
| }, |
| { |
| "index": 39, |
| "attention_type": "standard", |
| "ffn_type": "moe", |
| "num_experts": 8, |
| "top_k": 0, |
| "has_shared_expert": true, |
| "d_model": 2048, |
| "intermediate_size": 512, |
| "module_path": "model.layers.39" |
| } |
| ], |
| "moe_layer_indices": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 23, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 29, |
| 30, |
| 31, |
| 32, |
| 33, |
| 34, |
| 35, |
| 36, |
| 37, |
| 38, |
| 39 |
| ], |
| "dense_layer_indices": [], |
| "attention_layer_indices": [ |
| 3, |
| 7, |
| 11, |
| 15, |
| 19, |
| 23, |
| 27, |
| 31, |
| 35, |
| 39 |
| ], |
| "deltanet_layer_indices": [ |
| 0, |
| 1, |
| 2, |
| 4, |
| 5, |
| 6, |
| 8, |
| 9, |
| 10, |
| 12, |
| 13, |
| 14, |
| 16, |
| 17, |
| 18, |
| 20, |
| 21, |
| 22, |
| 24, |
| 25, |
| 26, |
| 28, |
| 29, |
| 30, |
| 32, |
| 33, |
| 34, |
| 36, |
| 37, |
| 38 |
| ], |
| "has_vision": true, |
| "vision_num_layers": 27, |
| "vision_hidden_size": 1152, |
| "vision_intermediate_size": 4304, |
| "vision_out_hidden_size": 2048, |
| "vision_layers": [ |
| { |
| "index": 0, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.0" |
| }, |
| { |
| "index": 1, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.1" |
| }, |
| { |
| "index": 2, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.2" |
| }, |
| { |
| "index": 3, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.3" |
| }, |
| { |
| "index": 4, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.4" |
| }, |
| { |
| "index": 5, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.5" |
| }, |
| { |
| "index": 6, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.6" |
| }, |
| { |
| "index": 7, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.7" |
| }, |
| { |
| "index": 8, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.8" |
| }, |
| { |
| "index": 9, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.9" |
| }, |
| { |
| "index": 10, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.10" |
| }, |
| { |
| "index": 11, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.11" |
| }, |
| { |
| "index": 12, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.12" |
| }, |
| { |
| "index": 13, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.13" |
| }, |
| { |
| "index": 14, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.14" |
| }, |
| { |
| "index": 15, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.15" |
| }, |
| { |
| "index": 16, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.16" |
| }, |
| { |
| "index": 17, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.17" |
| }, |
| { |
| "index": 18, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.18" |
| }, |
| { |
| "index": 19, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.19" |
| }, |
| { |
| "index": 20, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.20" |
| }, |
| { |
| "index": 21, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.21" |
| }, |
| { |
| "index": 22, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.22" |
| }, |
| { |
| "index": 23, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.23" |
| }, |
| { |
| "index": 24, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.24" |
| }, |
| { |
| "index": 25, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.25" |
| }, |
| { |
| "index": 26, |
| "d_vision": 1152, |
| "intermediate_size": 4304, |
| "num_heads": 16, |
| "module_path": "model.visual.blocks.26" |
| } |
| ] |
| } |
