voxpm2-ncnn / residual_decoder_kv.vulkan.ncnn.param
lyrin's picture
update for full support fp16 inference
ea65250 verified
Raw
History Blame Contribute Delete
20.9 kB
7767517
247 325
Input in0 0 1 in0
Split splitncnn_0 1 2 in0 1 2
Input in1 0 1 in1
Input kv_cache 0 16 cache_k0 cache_v0 cache_k1 cache_v1 cache_k2 cache_v2 cache_k3 cache_v3 cache_k4 cache_v4 cache_k5 cache_v5 cache_k6 cache_v6 cache_k7 cache_v7
Split splitncnn_1 1 8 in1 4 5 6 7 8 9 10 11
VoxCPM2DTypeAdapter rmsn_56_dtype_adapter 1 1 2 2_dtype_adapter_0
RMSNorm rmsn_56 1 1 2_dtype_adapter_0 12_dtype_rms_0 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_56_cast_out_fp16 1 1 12_dtype_rms_0 12 0=1 1=2
Split splitncnn_2 1 3 12 13 14 15
Gemm gemm_0 1 1 15 16 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
Gemm gemm_1 1 1 14 17 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Gemm gemm_2 1 1 13 18 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Reshape reshape_81 1 1 16 19 0=128 1=16 2=-1
Permute transpose_113 1 1 19 20 0=2
Reshape reshape_82 1 1 17 21 0=128 1=2 2=-1
Permute transpose_114 1 1 21 22 0=2
Reshape reshape_83 1 1 18 23 0=128 1=2 2=-1
Permute transpose_115 1 1 23 24 0=2
SDPA sdpa_145 6 3 20 22 24 11 cache_k0 cache_v0 25 out_cache_k0 out_cache_v0 5=1 6=0.08838835 7=1
Permute transpose_116 1 1 25 26 0=2
Reshape reshape_84 1 1 26 27 0=2048 1=-1
Gemm gemm_3 1 1 27 28 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
BinaryOp add_0 2 1 1 28 29 0=0
Split splitncnn_3 1 2 29 30 31
VoxCPM2DTypeAdapter rmsn_57_dtype_adapter 1 1 31 31_dtype_adapter_1
RMSNorm rmsn_57 1 1 31_dtype_adapter_1 32_dtype_rms_1 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_57_cast_out_fp16 1 1 32_dtype_rms_1 32 0=1 1=2
Split splitncnn_4 1 2 32 33 34
Gemm gemm_4 1 1 34 35 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
Swish silu_73 1 1 35 36
Gemm gemm_5 1 1 33 37 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
BinaryOp mul_1 2 1 36 37 38 0=2
Gemm gemm_6 1 1 38 39 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=6144
BinaryOp add_2 2 1 30 39 40 0=0
Split splitncnn_5 1 2 40 41 42
VoxCPM2DTypeAdapter rmsn_58_dtype_adapter 1 1 42 42_dtype_adapter_2
RMSNorm rmsn_58 1 1 42_dtype_adapter_2 43_dtype_rms_2 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_58_cast_out_fp16 1 1 43_dtype_rms_2 43 0=1 1=2
Split splitncnn_6 1 3 43 44 45 46
Gemm gemm_7 1 1 46 47 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
Gemm gemm_8 1 1 45 48 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Gemm gemm_9 1 1 44 49 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Reshape reshape_85 1 1 47 50 0=128 1=16 2=-1
Permute transpose_117 1 1 50 51 0=2
Reshape reshape_86 1 1 48 52 0=128 1=2 2=-1
Permute transpose_118 1 1 52 53 0=2
Reshape reshape_87 1 1 49 54 0=128 1=2 2=-1
Permute transpose_119 1 1 54 55 0=2
SDPA sdpa_146 6 3 51 53 55 10 cache_k1 cache_v1 56 out_cache_k1 out_cache_v1 5=1 6=0.08838835 7=1
Permute transpose_120 1 1 56 57 0=2
Reshape reshape_88 1 1 57 58 0=2048 1=-1
Gemm gemm_10 1 1 58 59 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
BinaryOp add_3 2 1 41 59 60 0=0
Split splitncnn_7 1 2 60 61 62
VoxCPM2DTypeAdapter rmsn_59_dtype_adapter 1 1 62 62_dtype_adapter_3
RMSNorm rmsn_59 1 1 62_dtype_adapter_3 63_dtype_rms_3 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_59_cast_out_fp16 1 1 63_dtype_rms_3 63 0=1 1=2
Split splitncnn_8 1 2 63 64 65
Gemm gemm_11 1 1 65 66 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
Swish silu_74 1 1 66 67
Gemm gemm_12 1 1 64 68 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
BinaryOp mul_4 2 1 67 68 69 0=2
Gemm gemm_13 1 1 69 70 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=6144
BinaryOp add_5 2 1 61 70 71 0=0
Split splitncnn_9 1 2 71 72 73
VoxCPM2DTypeAdapter rmsn_60_dtype_adapter 1 1 73 73_dtype_adapter_4
RMSNorm rmsn_60 1 1 73_dtype_adapter_4 74_dtype_rms_4 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_60_cast_out_fp16 1 1 74_dtype_rms_4 74 0=1 1=2
Split splitncnn_10 1 3 74 75 76 77
Gemm gemm_14 1 1 77 78 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
Gemm gemm_15 1 1 76 79 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Gemm gemm_16 1 1 75 80 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Reshape reshape_89 1 1 78 81 0=128 1=16 2=-1
Permute transpose_121 1 1 81 82 0=2
Reshape reshape_90 1 1 79 83 0=128 1=2 2=-1
Permute transpose_122 1 1 83 84 0=2
Reshape reshape_91 1 1 80 85 0=128 1=2 2=-1
Permute transpose_123 1 1 85 86 0=2
SDPA sdpa_147 6 3 82 84 86 9 cache_k2 cache_v2 87 out_cache_k2 out_cache_v2 5=1 6=0.08838835 7=1
Permute transpose_124 1 1 87 88 0=2
Reshape reshape_92 1 1 88 89 0=2048 1=-1
Gemm gemm_17 1 1 89 90 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
BinaryOp add_6 2 1 72 90 91 0=0
Split splitncnn_11 1 2 91 92 93
VoxCPM2DTypeAdapter rmsn_61_dtype_adapter 1 1 93 93_dtype_adapter_5
RMSNorm rmsn_61 1 1 93_dtype_adapter_5 94_dtype_rms_5 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_61_cast_out_fp16 1 1 94_dtype_rms_5 94 0=1 1=2
Split splitncnn_12 1 2 94 95 96
Gemm gemm_18 1 1 96 97 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
Swish silu_75 1 1 97 98
Gemm gemm_19 1 1 95 99 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
BinaryOp mul_7 2 1 98 99 100 0=2
Gemm gemm_20 1 1 100 101 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=6144
BinaryOp add_8 2 1 92 101 102 0=0
Split splitncnn_13 1 2 102 103 104
VoxCPM2DTypeAdapter rmsn_62_dtype_adapter 1 1 104 104_dtype_adapter_6
RMSNorm rmsn_62 1 1 104_dtype_adapter_6 105_dtype_rms_6 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_62_cast_out_fp16 1 1 105_dtype_rms_6 105 0=1 1=2
Split splitncnn_14 1 3 105 106 107 108
Gemm gemm_21 1 1 108 109 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
Gemm gemm_22 1 1 107 110 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Gemm gemm_23 1 1 106 111 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Reshape reshape_93 1 1 109 112 0=128 1=16 2=-1
Permute transpose_125 1 1 112 113 0=2
Reshape reshape_94 1 1 110 114 0=128 1=2 2=-1
Permute transpose_126 1 1 114 115 0=2
Reshape reshape_95 1 1 111 116 0=128 1=2 2=-1
Permute transpose_127 1 1 116 117 0=2
SDPA sdpa_148 6 3 113 115 117 8 cache_k3 cache_v3 118 out_cache_k3 out_cache_v3 5=1 6=0.08838835 7=1
Permute transpose_128 1 1 118 119 0=2
Reshape reshape_96 1 1 119 120 0=2048 1=-1
Gemm gemm_24 1 1 120 121 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
BinaryOp add_9 2 1 103 121 122 0=0
Split splitncnn_15 1 2 122 123 124
VoxCPM2DTypeAdapter rmsn_63_dtype_adapter 1 1 124 124_dtype_adapter_7
RMSNorm rmsn_63 1 1 124_dtype_adapter_7 125_dtype_rms_7 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_63_cast_out_fp16 1 1 125_dtype_rms_7 125 0=1 1=2
Split splitncnn_16 1 2 125 126 127
Gemm gemm_25 1 1 127 128 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
Swish silu_76 1 1 128 129
Gemm gemm_26 1 1 126 130 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
BinaryOp mul_10 2 1 129 130 131 0=2
Gemm gemm_27 1 1 131 132 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=6144
BinaryOp add_11 2 1 123 132 133 0=0
Split splitncnn_17 1 2 133 134 135
VoxCPM2DTypeAdapter rmsn_64_dtype_adapter 1 1 135 135_dtype_adapter_8
RMSNorm rmsn_64 1 1 135_dtype_adapter_8 136_dtype_rms_8 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_64_cast_out_fp16 1 1 136_dtype_rms_8 136 0=1 1=2
Split splitncnn_18 1 3 136 137 138 139
Gemm gemm_28 1 1 139 140 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
Gemm gemm_29 1 1 138 141 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Gemm gemm_30 1 1 137 142 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Reshape reshape_97 1 1 140 143 0=128 1=16 2=-1
Permute transpose_129 1 1 143 144 0=2
Reshape reshape_98 1 1 141 145 0=128 1=2 2=-1
Permute transpose_130 1 1 145 146 0=2
Reshape reshape_99 1 1 142 147 0=128 1=2 2=-1
Permute transpose_131 1 1 147 148 0=2
SDPA sdpa_149 6 3 144 146 148 7 cache_k4 cache_v4 149 out_cache_k4 out_cache_v4 5=1 6=0.08838835 7=1
Permute transpose_132 1 1 149 150 0=2
Reshape reshape_100 1 1 150 151 0=2048 1=-1
Gemm gemm_31 1 1 151 152 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
BinaryOp add_12 2 1 134 152 153 0=0
Split splitncnn_19 1 2 153 154 155
VoxCPM2DTypeAdapter rmsn_65_dtype_adapter 1 1 155 155_dtype_adapter_9
RMSNorm rmsn_65 1 1 155_dtype_adapter_9 156_dtype_rms_9 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_65_cast_out_fp16 1 1 156_dtype_rms_9 156 0=1 1=2
Split splitncnn_20 1 2 156 157 158
Gemm gemm_32 1 1 158 159 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
Swish silu_77 1 1 159 160
Gemm gemm_33 1 1 157 161 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
BinaryOp mul_13 2 1 160 161 162 0=2
Gemm gemm_34 1 1 162 163 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=6144
BinaryOp add_14 2 1 154 163 164 0=0
Split splitncnn_21 1 2 164 165 166
VoxCPM2DTypeAdapter rmsn_66_dtype_adapter 1 1 166 166_dtype_adapter_10
RMSNorm rmsn_66 1 1 166_dtype_adapter_10 167_dtype_rms_10 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_66_cast_out_fp16 1 1 167_dtype_rms_10 167 0=1 1=2
Split splitncnn_22 1 3 167 168 169 170
Gemm gemm_35 1 1 170 171 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
Gemm gemm_36 1 1 169 172 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Gemm gemm_37 1 1 168 173 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Reshape reshape_101 1 1 171 174 0=128 1=16 2=-1
Permute transpose_133 1 1 174 175 0=2
Reshape reshape_102 1 1 172 176 0=128 1=2 2=-1
Permute transpose_134 1 1 176 177 0=2
Reshape reshape_103 1 1 173 178 0=128 1=2 2=-1
Permute transpose_135 1 1 178 179 0=2
SDPA sdpa_150 6 3 175 177 179 6 cache_k5 cache_v5 180 out_cache_k5 out_cache_v5 5=1 6=0.08838835 7=1
Permute transpose_136 1 1 180 181 0=2
Reshape reshape_104 1 1 181 182 0=2048 1=-1
Gemm gemm_38 1 1 182 183 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
BinaryOp add_15 2 1 165 183 184 0=0
Split splitncnn_23 1 2 184 185 186
VoxCPM2DTypeAdapter rmsn_67_dtype_adapter 1 1 186 186_dtype_adapter_11
RMSNorm rmsn_67 1 1 186_dtype_adapter_11 187_dtype_rms_11 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_67_cast_out_fp16 1 1 187_dtype_rms_11 187 0=1 1=2
Split splitncnn_24 1 2 187 188 189
Gemm gemm_39 1 1 189 190 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
Swish silu_78 1 1 190 191
Gemm gemm_40 1 1 188 192 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
BinaryOp mul_16 2 1 191 192 193 0=2
Gemm gemm_41 1 1 193 194 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=6144
BinaryOp add_17 2 1 185 194 195 0=0
Split splitncnn_25 1 2 195 196 197
VoxCPM2DTypeAdapter rmsn_68_dtype_adapter 1 1 197 197_dtype_adapter_12
RMSNorm rmsn_68 1 1 197_dtype_adapter_12 198_dtype_rms_12 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_68_cast_out_fp16 1 1 198_dtype_rms_12 198 0=1 1=2
Split splitncnn_26 1 3 198 199 200 201
Gemm gemm_42 1 1 201 202 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
Gemm gemm_43 1 1 200 203 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Gemm gemm_44 1 1 199 204 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Reshape reshape_105 1 1 202 205 0=128 1=16 2=-1
Permute transpose_137 1 1 205 206 0=2
Reshape reshape_106 1 1 203 207 0=128 1=2 2=-1
Permute transpose_138 1 1 207 208 0=2
Reshape reshape_107 1 1 204 209 0=128 1=2 2=-1
Permute transpose_139 1 1 209 210 0=2
SDPA sdpa_151 6 3 206 208 210 5 cache_k6 cache_v6 211 out_cache_k6 out_cache_v6 5=1 6=0.08838835 7=1
Permute transpose_140 1 1 211 212 0=2
Reshape reshape_108 1 1 212 213 0=2048 1=-1
Gemm gemm_45 1 1 213 214 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
BinaryOp add_18 2 1 196 214 215 0=0
Split splitncnn_27 1 2 215 216 217
VoxCPM2DTypeAdapter rmsn_69_dtype_adapter 1 1 217 217_dtype_adapter_13
RMSNorm rmsn_69 1 1 217_dtype_adapter_13 218_dtype_rms_13 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_69_cast_out_fp16 1 1 218_dtype_rms_13 218 0=1 1=2
Split splitncnn_28 1 2 218 219 220
Gemm gemm_46 1 1 220 221 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
Swish silu_79 1 1 221 222
Gemm gemm_47 1 1 219 223 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
BinaryOp mul_19 2 1 222 223 224 0=2
Gemm gemm_48 1 1 224 225 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=6144
BinaryOp add_20 2 1 216 225 226 0=0
Split splitncnn_29 1 2 226 227 228
VoxCPM2DTypeAdapter rmsn_70_dtype_adapter 1 1 228 228_dtype_adapter_14
RMSNorm rmsn_70 1 1 228_dtype_adapter_14 229_dtype_rms_14 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_70_cast_out_fp16 1 1 229_dtype_rms_14 229 0=1 1=2
Split splitncnn_30 1 3 229 230 231 232
Gemm gemm_49 1 1 232 233 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
Gemm gemm_50 1 1 231 234 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Gemm gemm_51 1 1 230 235 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=256 9=2048
Reshape reshape_109 1 1 233 236 0=128 1=16 2=-1
Permute transpose_141 1 1 236 237 0=2
Reshape reshape_110 1 1 234 238 0=128 1=2 2=-1
Permute transpose_142 1 1 238 239 0=2
Reshape reshape_111 1 1 235 240 0=128 1=2 2=-1
Permute transpose_143 1 1 240 241 0=2
SDPA sdpa_152 6 3 237 239 241 4 cache_k7 cache_v7 242 out_cache_k7 out_cache_v7 5=1 6=0.08838835 7=1
Permute transpose_144 1 1 242 243 0=2
Reshape reshape_112 1 1 243 244 0=2048 1=-1
Gemm gemm_52 1 1 244 245 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=2048
BinaryOp add_21 2 1 227 245 246 0=0
Split splitncnn_31 1 2 246 247 248
VoxCPM2DTypeAdapter rmsn_71_dtype_adapter 1 1 248 248_dtype_adapter_15
RMSNorm rmsn_71 1 1 248_dtype_adapter_15 249_dtype_rms_15 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_71_cast_out_fp16 1 1 249_dtype_rms_15 249 0=1 1=2
Split splitncnn_32 1 2 249 250 251
Gemm gemm_53 1 1 251 252 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
Swish silu_80 1 1 252 253
Gemm gemm_54 1 1 250 254 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=6144 9=2048
BinaryOp mul_22 2 1 253 254 255 0=2
Gemm gemm_55 1 1 255 256 10=-1 2=0 3=1 4=0 5=1 6=1 7=0 8=2048 9=6144
BinaryOp add_23 2 1 247 256 257 0=0
VoxCPM2DTypeAdapter rmsn_72_dtype_adapter 1 1 257 257_dtype_adapter_16
RMSNorm rmsn_72 1 1 257_dtype_adapter_16 out0_dtype_rms_16 0=2048 1=1.000000e-5 2=1 31=3
Cast rmsn_72_cast_out_fp16 1 1 out0_dtype_rms_16 out0 0=1 1=2