File size: 9,068 Bytes
2c1a8e3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 | SenseNovaLoopForConditionalGeneration(
(model): Qwen3_5Model(
(visual): Qwen3_5VisionModel(
(patch_embed): Qwen3_5VisionPatchEmbed(
(proj): Conv3d(3, 64, kernel_size=(2, 16, 16), stride=(2, 16, 16))
)
(pos_embed): Embedding(2304, 64)
(rotary_pos_emb): Qwen3_5VisionRotaryEmbedding()
(blocks): ModuleList(
(0-3): 4 x Qwen3_5VisionBlock(
(norm1): LayerNorm((64,), eps=1e-06, elementwise_affine=True, bias=True)
(norm2): LayerNorm((64,), eps=1e-06, elementwise_affine=True, bias=True)
(attn): Qwen3_5VisionAttention(
(qkv): Linear(in_features=64, out_features=192, bias=True)
(proj): Linear(in_features=64, out_features=64, bias=True)
)
(mlp): Qwen3_5VisionMLP(
(linear_fc1): Linear(in_features=64, out_features=128, bias=True)
(linear_fc2): Linear(in_features=128, out_features=64, bias=True)
(act_fn): GELUTanh()
)
)
)
(merger): Qwen3_5VisionPatchMerger(
(norm): LayerNorm((64,), eps=1e-06, elementwise_affine=True, bias=True)
(linear_fc1): Linear(in_features=256, out_features=256, bias=True)
(act_fn): GELU(approximate='none')
(linear_fc2): Linear(in_features=256, out_features=8, bias=True)
)
)
(language_model): SenseNovaLoopTextModel(
(embed_tokens): Embedding(248320, 8, padding_idx=248044)
(layers): ModuleList(
(0-2): 3 x Qwen3_5DecoderLayer(
(linear_attn): Qwen3_5GatedDeltaNet(
(conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
(norm): Qwen3_5RMSNormGated()
(out_proj): Linear(in_features=256, out_features=8, bias=False)
(in_proj_qkv): Linear(in_features=8, out_features=512, bias=False)
(in_proj_z): Linear(in_features=8, out_features=256, bias=False)
(in_proj_b): Linear(in_features=8, out_features=8, bias=False)
(in_proj_a): Linear(in_features=8, out_features=8, bias=False)
)
(mlp): Qwen3_5MLP(
(gate_proj): Linear(in_features=8, out_features=32, bias=False)
(up_proj): Linear(in_features=8, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=8, bias=False)
(act_fn): SiLUActivation()
)
(input_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
(post_attention_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
)
(3): Qwen3_5DecoderLayer(
(self_attn): Qwen3_5Attention(
(q_proj): Linear(in_features=8, out_features=512, bias=False)
(k_proj): Linear(in_features=8, out_features=128, bias=False)
(v_proj): Linear(in_features=8, out_features=128, bias=False)
(o_proj): Linear(in_features=256, out_features=8, bias=False)
(q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
(k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
)
(mlp): Qwen3_5MLP(
(gate_proj): Linear(in_features=8, out_features=32, bias=False)
(up_proj): Linear(in_features=8, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=8, bias=False)
(act_fn): SiLUActivation()
)
(input_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
(post_attention_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
)
(4-6): 3 x MHCDecoderLayer(
(linear_attn): Qwen3_5GatedDeltaNet(
(conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
(norm): Qwen3_5RMSNormGated()
(out_proj): Linear(in_features=256, out_features=8, bias=False)
(in_proj_qkv): Linear(in_features=8, out_features=512, bias=False)
(in_proj_z): Linear(in_features=8, out_features=256, bias=False)
(in_proj_b): Linear(in_features=8, out_features=8, bias=False)
(in_proj_a): Linear(in_features=8, out_features=8, bias=False)
)
(mlp): Qwen3_5MLP(
(gate_proj): Linear(in_features=8, out_features=32, bias=False)
(up_proj): Linear(in_features=8, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=8, bias=False)
(act_fn): SiLUActivation()
)
(input_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
(post_attention_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
(mhc_attn): MHCRouter()
(mhc_ffn): MHCRouter()
)
(7): MHCDecoderLayer(
(self_attn): Qwen3_5Attention(
(q_proj): Linear(in_features=8, out_features=512, bias=False)
(k_proj): Linear(in_features=8, out_features=128, bias=False)
(v_proj): Linear(in_features=8, out_features=128, bias=False)
(o_proj): Linear(in_features=256, out_features=8, bias=False)
(q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
(k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
)
(mlp): Qwen3_5MLP(
(gate_proj): Linear(in_features=8, out_features=32, bias=False)
(up_proj): Linear(in_features=8, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=8, bias=False)
(act_fn): SiLUActivation()
)
(input_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
(post_attention_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
(mhc_attn): MHCRouter()
(mhc_ffn): MHCRouter()
)
(8-10): 3 x Qwen3_5DecoderLayer(
(linear_attn): Qwen3_5GatedDeltaNet(
(conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
(norm): Qwen3_5RMSNormGated()
(out_proj): Linear(in_features=256, out_features=8, bias=False)
(in_proj_qkv): Linear(in_features=8, out_features=512, bias=False)
(in_proj_z): Linear(in_features=8, out_features=256, bias=False)
(in_proj_b): Linear(in_features=8, out_features=8, bias=False)
(in_proj_a): Linear(in_features=8, out_features=8, bias=False)
)
(mlp): Qwen3_5MLP(
(gate_proj): Linear(in_features=8, out_features=32, bias=False)
(up_proj): Linear(in_features=8, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=8, bias=False)
(act_fn): SiLUActivation()
)
(input_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
(post_attention_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
)
(11): Qwen3_5DecoderLayer(
(self_attn): Qwen3_5Attention(
(q_proj): Linear(in_features=8, out_features=512, bias=False)
(k_proj): Linear(in_features=8, out_features=128, bias=False)
(v_proj): Linear(in_features=8, out_features=128, bias=False)
(o_proj): Linear(in_features=256, out_features=8, bias=False)
(q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
(k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
)
(mlp): Qwen3_5MLP(
(gate_proj): Linear(in_features=8, out_features=32, bias=False)
(up_proj): Linear(in_features=8, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=8, bias=False)
(act_fn): SiLUActivation()
)
(input_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
(post_attention_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
)
)
(norm): Qwen3_5RMSNorm((8,), eps=1e-06)
(rotary_emb): Qwen3_5TextRotaryEmbedding()
)
)
(lm_head): Linear(in_features=8, out_features=248320, bias=False)
(mtp): ModuleDict(
(pre_fc_norm_embedding): RMSNorm((8,), eps=None, elementwise_affine=True)
(fc): Linear(in_features=16, out_features=8, bias=False)
(layers): ModuleList(
(0): Qwen3_5DecoderLayer(
(self_attn): Qwen3_5Attention(
(q_proj): Linear(in_features=8, out_features=512, bias=False)
(k_proj): Linear(in_features=8, out_features=128, bias=False)
(v_proj): Linear(in_features=8, out_features=128, bias=False)
(o_proj): Linear(in_features=256, out_features=8, bias=False)
(q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
(k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
)
(mlp): Qwen3_5MLP(
(gate_proj): Linear(in_features=8, out_features=32, bias=False)
(up_proj): Linear(in_features=8, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=8, bias=False)
(act_fn): SiLUActivation()
)
(input_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
(post_attention_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
)
)
(norm): RMSNorm((8,), eps=None, elementwise_affine=True)
(pre_fc_norm_hidden): RMSNorm((8,), eps=None, elementwise_affine=True)
)
) |