{
  "model_name": "Qwen/Qwen3.5-35B-A3B-FP8",
  "num_layers": 40,
  "d_model": 2048,
  "vocab_size": 248320,
  "layers": [
    {
      "index": 0,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.0"
    },
    {
      "index": 1,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.1"
    },
    {
      "index": 2,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.2"
    },
    {
      "index": 3,
      "attention_type": "standard",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.3"
    },
    {
      "index": 4,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.4"
    },
    {
      "index": 5,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.5"
    },
    {
      "index": 6,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.6"
    },
    {
      "index": 7,
      "attention_type": "standard",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.7"
    },
    {
      "index": 8,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.8"
    },
    {
      "index": 9,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.9"
    },
    {
      "index": 10,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.10"
    },
    {
      "index": 11,
      "attention_type": "standard",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.11"
    },
    {
      "index": 12,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.12"
    },
    {
      "index": 13,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.13"
    },
    {
      "index": 14,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.14"
    },
    {
      "index": 15,
      "attention_type": "standard",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.15"
    },
    {
      "index": 16,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.16"
    },
    {
      "index": 17,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.17"
    },
    {
      "index": 18,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.18"
    },
    {
      "index": 19,
      "attention_type": "standard",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.19"
    },
    {
      "index": 20,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.20"
    },
    {
      "index": 21,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.21"
    },
    {
      "index": 22,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.22"
    },
    {
      "index": 23,
      "attention_type": "standard",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.23"
    },
    {
      "index": 24,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.24"
    },
    {
      "index": 25,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.25"
    },
    {
      "index": 26,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.26"
    },
    {
      "index": 27,
      "attention_type": "standard",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.27"
    },
    {
      "index": 28,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.28"
    },
    {
      "index": 29,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.29"
    },
    {
      "index": 30,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.30"
    },
    {
      "index": 31,
      "attention_type": "standard",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.31"
    },
    {
      "index": 32,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.32"
    },
    {
      "index": 33,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.33"
    },
    {
      "index": 34,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.34"
    },
    {
      "index": 35,
      "attention_type": "standard",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.35"
    },
    {
      "index": 36,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.36"
    },
    {
      "index": 37,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.37"
    },
    {
      "index": 38,
      "attention_type": "gated_deltanet",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.38"
    },
    {
      "index": 39,
      "attention_type": "standard",
      "ffn_type": "moe",
      "num_experts": 8,
      "top_k": 0,
      "has_shared_expert": true,
      "d_model": 2048,
      "intermediate_size": 512,
      "module_path": "model.layers.39"
    }
  ],
  "moe_layer_indices": [
    0,
    1,
    2,
    3,
    4,
    5,
    6,
    7,
    8,
    9,
    10,
    11,
    12,
    13,
    14,
    15,
    16,
    17,
    18,
    19,
    20,
    21,
    22,
    23,
    24,
    25,
    26,
    27,
    28,
    29,
    30,
    31,
    32,
    33,
    34,
    35,
    36,
    37,
    38,
    39
  ],
  "dense_layer_indices": [],
  "attention_layer_indices": [
    3,
    7,
    11,
    15,
    19,
    23,
    27,
    31,
    35,
    39
  ],
  "deltanet_layer_indices": [
    0,
    1,
    2,
    4,
    5,
    6,
    8,
    9,
    10,
    12,
    13,
    14,
    16,
    17,
    18,
    20,
    21,
    22,
    24,
    25,
    26,
    28,
    29,
    30,
    32,
    33,
    34,
    36,
    37,
    38
  ],
  "has_vision": true,
  "vision_num_layers": 27,
  "vision_hidden_size": 1152,
  "vision_intermediate_size": 4304,
  "vision_out_hidden_size": 2048,
  "vision_layers": [
    {
      "index": 0,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.0"
    },
    {
      "index": 1,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.1"
    },
    {
      "index": 2,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.2"
    },
    {
      "index": 3,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.3"
    },
    {
      "index": 4,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.4"
    },
    {
      "index": 5,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.5"
    },
    {
      "index": 6,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.6"
    },
    {
      "index": 7,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.7"
    },
    {
      "index": 8,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.8"
    },
    {
      "index": 9,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.9"
    },
    {
      "index": 10,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.10"
    },
    {
      "index": 11,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.11"
    },
    {
      "index": 12,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.12"
    },
    {
      "index": 13,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.13"
    },
    {
      "index": 14,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.14"
    },
    {
      "index": 15,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.15"
    },
    {
      "index": 16,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.16"
    },
    {
      "index": 17,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.17"
    },
    {
      "index": 18,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.18"
    },
    {
      "index": 19,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.19"
    },
    {
      "index": 20,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.20"
    },
    {
      "index": 21,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.21"
    },
    {
      "index": 22,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.22"
    },
    {
      "index": 23,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.23"
    },
    {
      "index": 24,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.24"
    },
    {
      "index": 25,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.25"
    },
    {
      "index": 26,
      "d_vision": 1152,
      "intermediate_size": 4304,
      "num_heads": 16,
      "module_path": "model.visual.blocks.26"
    }
  ]
}