Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+output.png filter=lfs diff=lfs merge=lfs -text

net_capemb.ini

@@ -0,0 +1,23 @@
+[train]
+load_net    = 1
+load_file=z_image_turbo_transformer_cap_embedder.cccc.bin
+train_epochs = 0
+Batch       = 1
+gpu         = 1
+data_type = half
+output_log  = 0
+need_free_mem = 0
+
+[net]
+structure='
+B=1;
+X  = Matrix(2560, T, 1, B);
+Ws = MatrixWithName("rmsn_1.weight", 2560, 1);
+Xn = rmsNorm(X, Ws, 1e-5);
+W  = MatrixWithName("gemm_0.weight", 3840, 2560, 1, 1);
+b  = MatrixWithName("gemm_0.bias", 3840, 1, 1, 1);
+Y  = batchedMul(W, Xn) + b;
+setXY(X, Y);
+'
+
+

net_ctx.ini

@@ -0,0 +1,67 @@
+[train]
+load_net    = 1
+load_file=z_image_turbo_transformer_context_refiner.cccc.bin
+train_epochs = 0
+Batch       = 1
+gpu         = 1
+data_type = half
+output_log  = 0
+need_free_mem = 0
+
+[net]
+structure='
+B=1; D=3840; DD=3968; I=10240; hd=128; HB=30;
+Xw   = Matrix(DD, T, 1, B);
+X    = sliceW(Xw, 0, D);
+cos1 = sliceW(Xw, D, 64);
+sin1 = sliceW(Xw, D+64, 64);
+W_n16 = MatrixWithName("rmsn_16.weight", D, 1);
+W_q1  = MatrixWithName("gemm_0.weight",  D, D, 1, 1);
+W_k1  = MatrixWithName("gemm_1.weight",  D, D, 1, 1);
+W_v1  = MatrixWithName("gemm_2.weight",  D, D, 1, 1);
+W_n17 = MatrixWithName("rmsn_17.weight", hd, 1);
+W_n18 = MatrixWithName("rmsn_18.weight", hd, 1);
+W_o1  = MatrixWithName("gemm_3.weight",  D, D, 1, 1);
+W_n19 = MatrixWithName("rmsn_19.weight", D, 1);
+W_n20 = MatrixWithName("rmsn_20.weight", D, 1);
+W_g1  = MatrixWithName("gemm_4.weight",  I, D, 1, 1);
+W_u1  = MatrixWithName("gemm_5.weight",  I, D, 1, 1);
+W_d1  = MatrixWithName("gemm_6.weight",  D, I, 1, 1);
+W_n21 = MatrixWithName("rmsn_21.weight", D, 1);
+W_n22 = MatrixWithName("rmsn_22.weight", D, 1);
+W_q2  = MatrixWithName("gemm_7.weight",  D, D, 1, 1);
+W_k2  = MatrixWithName("gemm_8.weight",  D, D, 1, 1);
+W_v2  = MatrixWithName("gemm_9.weight",  D, D, 1, 1);
+W_n23 = MatrixWithName("rmsn_23.weight", hd, 1);
+W_n24 = MatrixWithName("rmsn_24.weight", hd, 1);
+W_o2  = MatrixWithName("gemm_10.weight", D, D, 1, 1);
+W_n25 = MatrixWithName("rmsn_25.weight", D, 1);
+W_n26 = MatrixWithName("rmsn_26.weight", D, 1);
+W_g2  = MatrixWithName("gemm_11.weight", I, D, 1, 1);
+W_u2  = MatrixWithName("gemm_12.weight", I, D, 1, 1);
+W_d2  = MatrixWithName("gemm_13.weight", D, I, 1, 1);
+W_n27 = MatrixWithName("rmsn_27.weight", D, 1);
+X1n = rmsNorm(X, W_n16, 1e-5);
+Q1r = permute(reshape(batchedMul(W_q1, X1n), {hd, HB, T, 1}), {0, 2, 3, 1});
+K1r = permute(reshape(batchedMul(W_k1, X1n), {hd, HB, T, 1}), {0, 2, 3, 1});
+V1r = permute(reshape(batchedMul(W_v1, X1n), {hd, HB, T, 1}), {0, 2, 3, 1});
+Q1n = rmsNorm(Q1r, W_n17, 1e-5);
+K1n = rmsNorm(K1r, W_n18, 1e-5);
+A1 = attention(rope2(Q1n, cos1, sin1), rope2(K1n, cos1, sin1), V1r, hd, 0);
+R1 = X + rmsNorm(batchedMul(W_o1, reshape(permute(A1, {0, 3, 1, 2}), {D, T, 1, 1})), W_n19, 1e-5);
+R1n = rmsNorm(R1, W_n20, 1e-5);
+R1b = R1 + rmsNorm(batchedMul(W_d1, elementMul(silu(batchedMul(W_g1, R1n)), batchedMul(W_u1, R1n))), W_n21, 1e-5);
+X2n = rmsNorm(R1b, W_n22, 1e-5);
+Q2r = permute(reshape(batchedMul(W_q2, X2n), {hd, HB, T, 1}), {0, 2, 3, 1});
+K2r = permute(reshape(batchedMul(W_k2, X2n), {hd, HB, T, 1}), {0, 2, 3, 1});
+V2r = permute(reshape(batchedMul(W_v2, X2n), {hd, HB, T, 1}), {0, 2, 3, 1});
+Q2n = rmsNorm(Q2r, W_n23, 1e-5);
+K2n = rmsNorm(K2r, W_n24, 1e-5);
+A2 = attention(rope2(Q2n, cos1, sin1), rope2(K2n, cos1, sin1), V2r, hd, 0);
+R2 = R1b + rmsNorm(batchedMul(W_o2, reshape(permute(A2, {0, 3, 1, 2}), {D, T, 1, 1})), W_n25, 1e-5);
+R2n = rmsNorm(R2, W_n26, 1e-5);
+Y = R2 + rmsNorm(batchedMul(W_d2, elementMul(silu(batchedMul(W_g2, R2n)), batchedMul(W_u2, R2n))), W_n27, 1e-5);
+setXY(Xw, Y);
+'
+
+

net_final.ini

@@ -0,0 +1,35 @@
+[train]
+load_net    = 1
+load_file=z_image_turbo_transformer_all_final_layer.cccc.bin
+train_epochs = 0
+Batch       = 1
+gpu         = 1
+data_type = float
+output_log  = 0
+need_free_mem = 0
+
+[net]
+structure='
+B=1; D=3840;
+X = Matrix(D, SEQ+1, 1, B);
+Xpm    = permute(X, {1, 0, 2, 3});
+t_slot = sliceW(Xpm, SEQ, 1);
+t_full = permute(t_slot, {1, 0, 2, 3});
+t256   = sliceW(t_full, 0, 256);
+seq_pm = sliceW(Xpm, 0, P);
+Xi     = permute(seq_pm, {1, 0, 2, 3});
+W0 = MatrixWithName("gemm_0.weight", D, 256, 1, 1);
+b0 = MatrixWithName("gemm_0.bias",   D, 1,   1, 1);
+scale  = batchedMul(W0, silu(t256)) + b0;
+setNeedLoadState(0);
+s_ln = Matrix(D, 1, 1, 1);
+b_ln = Matrix(D, 1, 1, 1);
+setNeedLoadState(1);
+Xi_ln  = layerNorm(Xi, s_ln, b_ln, 1e-6);
+Xi_mod = Xi_ln + elementMul(Xi_ln, scale);
+W1 = MatrixWithName("gemm_1.weight", 64, D, 1, 1);
+b1 = MatrixWithName("gemm_1.bias",   64, 1, 1, 1);
+Y  = batchedMul(W1, Xi_mod) + b1;
+setXY(X, Y);
+'
+

net_noise.ini

@@ -0,0 +1,106 @@
+[train]
+load_net    = 1
+load_file=z_image_turbo_transformer_noise_refiner.cccc.bin
+train_epochs = 0
+Batch       = 1
+gpu         = 1
+data_type = float
+output_log  = 0
+need_free_mem = 0
+
+[net]
+structure='
+B=1; D=3840; DD=3968; I=10240; hd=128; HB=30;
+Xin_n = Matrix(DD, P+1, 1, B);
+Xin_npm = permute(Xin_n, {1, 0, 2, 3});
+t_sn = sliceW(Xin_npm, P, 1);
+t_ful = permute(t_sn, {1, 0, 2, 3});
+t_emb_n = sliceW(t_ful, 0, 256);
+seq_npm = sliceW(Xin_npm, 0, P);
+Xn0_wide = permute(seq_npm, {1, 0, 2, 3});
+Xn0 = sliceW(Xn0_wide, 0, D);
+cosn = sliceW(Xn0_wide, D, 64);
+sinn = sliceW(Xn0_wide, D+64, 64);
+Wmod_n0 = MatrixWithName("gemm_0.weight", 15360, 256, 1, 1);
+bmod_n0 = MatrixWithName("gemm_0.bias", 15360, 1, 1, 1);
+modn0 = batchedMul(Wmod_n0, t_emb_n) + bmod_n0;
+sa_n0 = sliceW(modn0, 0, D);
+ga_n0 = tanh(sliceW(modn0, D, D));
+sf_n0 = sliceW(modn0, 2*D, D);
+gf_n0 = tanh(sliceW(modn0, 3*D, D));
+Wrnn22 = MatrixWithName("rmsn_22.weight", D, 1);
+Xnn0 = rmsNorm(Xn0, Wrnn22, 1e-5);
+Xmn0 = Xnn0 + elementMul(Xnn0, sa_n0);
+Wqn0 = MatrixWithName("gemm_1.weight", D, D, 1, 1);
+Wkn0 = MatrixWithName("gemm_2.weight", D, D, 1, 1);
+Wvn0 = MatrixWithName("gemm_3.weight", D, D, 1, 1);
+Wrnn23 = MatrixWithName("rmsn_23.weight", hd, 1);
+Wrnn24 = MatrixWithName("rmsn_24.weight", hd, 1);
+Wno0 = MatrixWithName("gemm_4.weight", D, D, 1, 1);
+Wrnn25 = MatrixWithName("rmsn_25.weight", D, 1);
+Qrn0 = permute(reshape(batchedMul(Wqn0, Xmn0), {hd, HB, P, 1}), {0, 2, 3, 1});
+Krn0 = permute(reshape(batchedMul(Wkn0, Xmn0), {hd, HB, P, 1}), {0, 2, 3, 1});
+Vrn0 = permute(reshape(batchedMul(Wvn0, Xmn0), {hd, HB, P, 1}), {0, 2, 3, 1});
+Qnn0 = rmsNorm(Qrn0, Wrnn23, 1e-5);
+Knn0 = rmsNorm(Krn0, Wrnn24, 1e-5);
+Atn0 = attention(rope2(Qnn0, cosn, sinn), rope2(Knn0, cosn, sinn), Vrn0, hd, 0);
+Aon0 = batchedMul(Wno0, reshape(permute(Atn0, {0, 3, 1, 2}), {D, P, 1, 1}));
+Ann0 = rmsNorm(Aon0, Wrnn25, 1e-5);
+Agn0 = elementMul(Ann0, ga_n0);
+Rxn0 = Xn0 + Agn0;
+Wrnn26 = MatrixWithName("rmsn_26.weight", D, 1);
+Wgn0 = MatrixWithName("gemm_5.weight", I, D, 1, 1);
+Wun0 = MatrixWithName("gemm_6.weight", I, D, 1, 1);
+Wdn0 = MatrixWithName("gemm_7.weight", D, I, 1, 1);
+Wrnn27 = MatrixWithName("rmsn_27.weight", D, 1);
+Rfn0 = rmsNorm(Rxn0, Wrnn26, 1e-5);
+Rfmn0 = Rfn0 + elementMul(Rfn0, sf_n0);
+Fgn0 = batchedMul(Wgn0, Rfmn0);
+Fun0 = batchedMul(Wun0, Rfmn0);
+Ffn0 = batchedMul(Wdn0, elementMul(silu(Fgn0), Fun0));
+Fnn0 = rmsNorm(Ffn0, Wrnn27, 1e-5);
+Fggn0 = elementMul(Fnn0, gf_n0);
+Xn1 = Rxn0 + Fggn0;
+Wmod_n1 = MatrixWithName("gemm_8.weight", 15360, 256, 1, 1);
+bmod_n1 = MatrixWithName("gemm_8.bias", 15360, 1, 1, 1);
+modn1 = batchedMul(Wmod_n1, t_emb_n) + bmod_n1;
+sa_n1 = sliceW(modn1, 0, D);
+ga_n1 = tanh(sliceW(modn1, D, D));
+sf_n1 = sliceW(modn1, 2*D, D);
+gf_n1 = tanh(sliceW(modn1, 3*D, D));
+Wrnn28 = MatrixWithName("rmsn_28.weight", D, 1);
+Xnn1 = rmsNorm(Xn1, Wrnn28, 1e-5);
+Xmn1 = Xnn1 + elementMul(Xnn1, sa_n1);
+Wqn1 = MatrixWithName("gemm_9.weight", D, D, 1, 1);
+Wkn1 = MatrixWithName("gemm_10.weight", D, D, 1, 1);
+Wvn1 = MatrixWithName("gemm_11.weight", D, D, 1, 1);
+Wrnn29 = MatrixWithName("rmsn_29.weight", hd, 1);
+Wrnn30 = MatrixWithName("rmsn_30.weight", hd, 1);
+Wno1 = MatrixWithName("gemm_12.weight", D, D, 1, 1);
+Wrnn31 = MatrixWithName("rmsn_31.weight", D, 1);
+Qrn1 = permute(reshape(batchedMul(Wqn1, Xmn1), {hd, HB, P, 1}), {0, 2, 3, 1});
+Krn1 = permute(reshape(batchedMul(Wkn1, Xmn1), {hd, HB, P, 1}), {0, 2, 3, 1});
+Vrn1 = permute(reshape(batchedMul(Wvn1, Xmn1), {hd, HB, P, 1}), {0, 2, 3, 1});
+Qnn1 = rmsNorm(Qrn1, Wrnn29, 1e-5);
+Knn1 = rmsNorm(Krn1, Wrnn30, 1e-5);
+Atn1 = attention(rope2(Qnn1, cosn, sinn), rope2(Knn1, cosn, sinn), Vrn1, hd, 0);
+Aon1 = batchedMul(Wno1, reshape(permute(Atn1, {0, 3, 1, 2}), {D, P, 1, 1}));
+Ann1 = rmsNorm(Aon1, Wrnn31, 1e-5);
+Agn1 = elementMul(Ann1, ga_n1);
+Rxn1 = Xn1 + Agn1;
+Wrnn32 = MatrixWithName("rmsn_32.weight", D, 1);
+Wgn1 = MatrixWithName("gemm_13.weight", I, D, 1, 1);
+Wun1 = MatrixWithName("gemm_14.weight", I, D, 1, 1);
+Wdn1 = MatrixWithName("gemm_15.weight", D, I, 1, 1);
+Wrnn33 = MatrixWithName("rmsn_33.weight", D, 1);
+Rfn1 = rmsNorm(Rxn1, Wrnn32, 1e-5);
+Rfmn1 = Rfn1 + elementMul(Rfn1, sf_n1);
+Fgn1 = batchedMul(Wgn1, Rfmn1);
+Fun1 = batchedMul(Wun1, Rfmn1);
+Ffn1 = batchedMul(Wdn1, elementMul(silu(Fgn1), Fun1));
+Fnn1 = rmsNorm(Ffn1, Wrnn33, 1e-5);
+Fggn1 = elementMul(Fnn1, gf_n1);
+Xn2 = Rxn1 + Fggn1;
+setXY(Xin_n, Xn2);
+'
+

net_temb.ini

@@ -0,0 +1,28 @@
+[train]
+load_net    = 1
+load_file=z_image_turbo_transformer_t_embedder.cccc.bin
+train_epochs = 0
+Batch       = 1
+gpu         = 1
+data_type = half
+output_log  = 0
+need_free_mem = 0
+
+[net]
+structure='
+B=1;
+freq = MatrixWithName("pnnx_fold_29_data.data", 128, 1);
+t_in = Matrix(1, 1, 1, B);
+t_sc = freq * t_in;
+cos_p = cos(t_sc);
+sin_p = sin(t_sc);
+embed = reshape(concat(cos_p, sin_p), {256, 1, 1, 1});
+W0 = MatrixWithName("gemm_0.weight", 1024, 256, 1, 1);
+b0 = MatrixWithName("gemm_0.bias", 1024, 1, 1, 1);
+h  = silu(batchedMul(W0, embed) + b0);
+W1 = MatrixWithName("gemm_1.weight", 256, 1024, 1, 1);
+b1 = MatrixWithName("gemm_1.bias", 256, 1, 1, 1);
+Y  = batchedMul(W1, h) + b1;
+setXY(t_in, Y);
+'
+

net_texenc.ini

@@ -0,0 +1,1141 @@
+[train]
+load_net    = 1
+load_file=z_image_turbo_text_encoder.cccc.bin
+train_epochs = 0
+Batch       = 1
+gpu         = 1
+data_type = half
+output_log  = 0
+need_free_mem = 0
+
+[net]
+structure='
+B=1; D=2560; Dq=4096; Dkv=1024; hd=128; HB=32; HkvB=8; HkvMul=4; I=9728; V=151936;
+W_emb = MatrixWithName("embed_280.weight", D, 1, 1, V);
+ids = MatrixF(T, 1, 1, B);
+X0 = embed(ids, W_emb);
+cos_te = ropeCosTbl(T, hd, 1000000.0);
+sin_te = ropeSinTbl(T, hd, 1000000.0);
+Wra0 = MatrixWithName("rmsn_281.weight", D, 1);
+Xna0 = rmsNorm(X0, Wra0, 1e-5);
+Wq0 = MatrixWithName("gemm_0.weight", Dq, D, 1, 1);
+Qh0 = permute(reshape(batchedMul(Wq0, Xna0), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn0 = MatrixWithName("rmsn_282.weight", hd, 1);
+Qn0 = rmsNorm(Qh0, Wqn0, 1e-5);
+Qr0 = rope(Qn0, cos_te, sin_te);
+Wk0 = MatrixWithName("gemm_1.weight", Dkv, D, 1, 1);
+Kh0 = permute(reshape(batchedMul(Wk0, Xna0), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn0 = MatrixWithName("rmsn_283.weight", hd, 1);
+Kn0 = rmsNorm(Kh0, Wkn0, 1e-5);
+Kr0 = rope(Kn0, cos_te, sin_te);
+Wv0 = MatrixWithName("gemm_2.weight", Dkv, D, 1, 1);
+V0 = permute(reshape(batchedMul(Wv0, Xna0), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc0 = reshapeBatch(Kr0, {hd, T, HkvB, 1});
+Krt0 = tile(Krc0, {1, 1, 1, HkvMul});
+Krp0 = permute(Krt0, {0, 1, 3, 2});
+Kt0 = reshapeBatch(Krp0, {hd, T, 1, HB});
+Vrc0 = reshapeBatch(V0, {hd, T, HkvB, 1});
+Vrt0 = tile(Vrc0, {1, 1, 1, HkvMul});
+Vrp0 = permute(Vrt0, {0, 1, 3, 2});
+Vt0 = reshapeBatch(Vrp0, {hd, T, 1, HB});
+A0 = attention(Qr0, Kt0, Vt0, hd, 1);
+Wo0 = MatrixWithName("gemm_3.weight", D, Dq, 1, 1);
+O0 = batchedMul(Wo0, reshape(permute(A0, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R0 = X0 + O0;
+Wrf0 = MatrixWithName("rmsn_284.weight", D, 1);
+Rfn0 = rmsNorm(R0, Wrf0, 1e-5);
+Wg0 = MatrixWithName("gemm_4.weight", I, D, 1, 1);
+Wu0 = MatrixWithName("gemm_5.weight", I, D, 1, 1);
+Wd0 = MatrixWithName("gemm_6.weight", D, I, 1, 1);
+X1 = R0 + batchedMul(Wd0, elementMul(silu(batchedMul(Wg0, Rfn0)), batchedMul(Wu0, Rfn0)));
+Wra1 = MatrixWithName("rmsn_285.weight", D, 1);
+Xna1 = rmsNorm(X1, Wra1, 1e-5);
+Wq1 = MatrixWithName("gemm_7.weight", Dq, D, 1, 1);
+Qh1 = permute(reshape(batchedMul(Wq1, Xna1), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn1 = MatrixWithName("rmsn_286.weight", hd, 1);
+Qn1 = rmsNorm(Qh1, Wqn1, 1e-5);
+Qr1 = rope(Qn1, cos_te, sin_te);
+Wk1 = MatrixWithName("gemm_8.weight", Dkv, D, 1, 1);
+Kh1 = permute(reshape(batchedMul(Wk1, Xna1), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn1 = MatrixWithName("rmsn_287.weight", hd, 1);
+Kn1 = rmsNorm(Kh1, Wkn1, 1e-5);
+Kr1 = rope(Kn1, cos_te, sin_te);
+Wv1 = MatrixWithName("gemm_9.weight", Dkv, D, 1, 1);
+V1 = permute(reshape(batchedMul(Wv1, Xna1), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc1 = reshapeBatch(Kr1, {hd, T, HkvB, 1});
+Krt1 = tile(Krc1, {1, 1, 1, HkvMul});
+Krp1 = permute(Krt1, {0, 1, 3, 2});
+Kt1 = reshapeBatch(Krp1, {hd, T, 1, HB});
+Vrc1 = reshapeBatch(V1, {hd, T, HkvB, 1});
+Vrt1 = tile(Vrc1, {1, 1, 1, HkvMul});
+Vrp1 = permute(Vrt1, {0, 1, 3, 2});
+Vt1 = reshapeBatch(Vrp1, {hd, T, 1, HB});
+A1 = attention(Qr1, Kt1, Vt1, hd, 1);
+Wo1 = MatrixWithName("gemm_10.weight", D, Dq, 1, 1);
+O1 = batchedMul(Wo1, reshape(permute(A1, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R1 = X1 + O1;
+Wrf1 = MatrixWithName("rmsn_288.weight", D, 1);
+Rfn1 = rmsNorm(R1, Wrf1, 1e-5);
+Wg1 = MatrixWithName("gemm_11.weight", I, D, 1, 1);
+Wu1 = MatrixWithName("gemm_12.weight", I, D, 1, 1);
+Wd1 = MatrixWithName("gemm_13.weight", D, I, 1, 1);
+X2 = R1 + batchedMul(Wd1, elementMul(silu(batchedMul(Wg1, Rfn1)), batchedMul(Wu1, Rfn1)));
+Wra2 = MatrixWithName("rmsn_289.weight", D, 1);
+Xna2 = rmsNorm(X2, Wra2, 1e-5);
+Wq2 = MatrixWithName("gemm_14.weight", Dq, D, 1, 1);
+Qh2 = permute(reshape(batchedMul(Wq2, Xna2), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn2 = MatrixWithName("rmsn_290.weight", hd, 1);
+Qn2 = rmsNorm(Qh2, Wqn2, 1e-5);
+Qr2 = rope(Qn2, cos_te, sin_te);
+Wk2 = MatrixWithName("gemm_15.weight", Dkv, D, 1, 1);
+Kh2 = permute(reshape(batchedMul(Wk2, Xna2), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn2 = MatrixWithName("rmsn_291.weight", hd, 1);
+Kn2 = rmsNorm(Kh2, Wkn2, 1e-5);
+Kr2 = rope(Kn2, cos_te, sin_te);
+Wv2 = MatrixWithName("gemm_16.weight", Dkv, D, 1, 1);
+V2 = permute(reshape(batchedMul(Wv2, Xna2), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc2 = reshapeBatch(Kr2, {hd, T, HkvB, 1});
+Krt2 = tile(Krc2, {1, 1, 1, HkvMul});
+Krp2 = permute(Krt2, {0, 1, 3, 2});
+Kt2 = reshapeBatch(Krp2, {hd, T, 1, HB});
+Vrc2 = reshapeBatch(V2, {hd, T, HkvB, 1});
+Vrt2 = tile(Vrc2, {1, 1, 1, HkvMul});
+Vrp2 = permute(Vrt2, {0, 1, 3, 2});
+Vt2 = reshapeBatch(Vrp2, {hd, T, 1, HB});
+A2 = attention(Qr2, Kt2, Vt2, hd, 1);
+Wo2 = MatrixWithName("gemm_17.weight", D, Dq, 1, 1);
+O2 = batchedMul(Wo2, reshape(permute(A2, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R2 = X2 + O2;
+Wrf2 = MatrixWithName("rmsn_292.weight", D, 1);
+Rfn2 = rmsNorm(R2, Wrf2, 1e-5);
+Wg2 = MatrixWithName("gemm_18.weight", I, D, 1, 1);
+Wu2 = MatrixWithName("gemm_19.weight", I, D, 1, 1);
+Wd2 = MatrixWithName("gemm_20.weight", D, I, 1, 1);
+X3 = R2 + batchedMul(Wd2, elementMul(silu(batchedMul(Wg2, Rfn2)), batchedMul(Wu2, Rfn2)));
+Wra3 = MatrixWithName("rmsn_293.weight", D, 1);
+Xna3 = rmsNorm(X3, Wra3, 1e-5);
+Wq3 = MatrixWithName("gemm_21.weight", Dq, D, 1, 1);
+Qh3 = permute(reshape(batchedMul(Wq3, Xna3), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn3 = MatrixWithName("rmsn_294.weight", hd, 1);
+Qn3 = rmsNorm(Qh3, Wqn3, 1e-5);
+Qr3 = rope(Qn3, cos_te, sin_te);
+Wk3 = MatrixWithName("gemm_22.weight", Dkv, D, 1, 1);
+Kh3 = permute(reshape(batchedMul(Wk3, Xna3), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn3 = MatrixWithName("rmsn_295.weight", hd, 1);
+Kn3 = rmsNorm(Kh3, Wkn3, 1e-5);
+Kr3 = rope(Kn3, cos_te, sin_te);
+Wv3 = MatrixWithName("gemm_23.weight", Dkv, D, 1, 1);
+V3 = permute(reshape(batchedMul(Wv3, Xna3), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc3 = reshapeBatch(Kr3, {hd, T, HkvB, 1});
+Krt3 = tile(Krc3, {1, 1, 1, HkvMul});
+Krp3 = permute(Krt3, {0, 1, 3, 2});
+Kt3 = reshapeBatch(Krp3, {hd, T, 1, HB});
+Vrc3 = reshapeBatch(V3, {hd, T, HkvB, 1});
+Vrt3 = tile(Vrc3, {1, 1, 1, HkvMul});
+Vrp3 = permute(Vrt3, {0, 1, 3, 2});
+Vt3 = reshapeBatch(Vrp3, {hd, T, 1, HB});
+A3 = attention(Qr3, Kt3, Vt3, hd, 1);
+Wo3 = MatrixWithName("gemm_24.weight", D, Dq, 1, 1);
+O3 = batchedMul(Wo3, reshape(permute(A3, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R3 = X3 + O3;
+Wrf3 = MatrixWithName("rmsn_296.weight", D, 1);
+Rfn3 = rmsNorm(R3, Wrf3, 1e-5);
+Wg3 = MatrixWithName("gemm_25.weight", I, D, 1, 1);
+Wu3 = MatrixWithName("gemm_26.weight", I, D, 1, 1);
+Wd3 = MatrixWithName("gemm_27.weight", D, I, 1, 1);
+X4 = R3 + batchedMul(Wd3, elementMul(silu(batchedMul(Wg3, Rfn3)), batchedMul(Wu3, Rfn3)));
+Wra4 = MatrixWithName("rmsn_297.weight", D, 1);
+Xna4 = rmsNorm(X4, Wra4, 1e-5);
+Wq4 = MatrixWithName("gemm_28.weight", Dq, D, 1, 1);
+Qh4 = permute(reshape(batchedMul(Wq4, Xna4), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn4 = MatrixWithName("rmsn_298.weight", hd, 1);
+Qn4 = rmsNorm(Qh4, Wqn4, 1e-5);
+Qr4 = rope(Qn4, cos_te, sin_te);
+Wk4 = MatrixWithName("gemm_29.weight", Dkv, D, 1, 1);
+Kh4 = permute(reshape(batchedMul(Wk4, Xna4), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn4 = MatrixWithName("rmsn_299.weight", hd, 1);
+Kn4 = rmsNorm(Kh4, Wkn4, 1e-5);
+Kr4 = rope(Kn4, cos_te, sin_te);
+Wv4 = MatrixWithName("gemm_30.weight", Dkv, D, 1, 1);
+V4 = permute(reshape(batchedMul(Wv4, Xna4), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc4 = reshapeBatch(Kr4, {hd, T, HkvB, 1});
+Krt4 = tile(Krc4, {1, 1, 1, HkvMul});
+Krp4 = permute(Krt4, {0, 1, 3, 2});
+Kt4 = reshapeBatch(Krp4, {hd, T, 1, HB});
+Vrc4 = reshapeBatch(V4, {hd, T, HkvB, 1});
+Vrt4 = tile(Vrc4, {1, 1, 1, HkvMul});
+Vrp4 = permute(Vrt4, {0, 1, 3, 2});
+Vt4 = reshapeBatch(Vrp4, {hd, T, 1, HB});
+A4 = attention(Qr4, Kt4, Vt4, hd, 1);
+Wo4 = MatrixWithName("gemm_31.weight", D, Dq, 1, 1);
+O4 = batchedMul(Wo4, reshape(permute(A4, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R4 = X4 + O4;
+Wrf4 = MatrixWithName("rmsn_300.weight", D, 1);
+Rfn4 = rmsNorm(R4, Wrf4, 1e-5);
+Wg4 = MatrixWithName("gemm_32.weight", I, D, 1, 1);
+Wu4 = MatrixWithName("gemm_33.weight", I, D, 1, 1);
+Wd4 = MatrixWithName("gemm_34.weight", D, I, 1, 1);
+X5 = R4 + batchedMul(Wd4, elementMul(silu(batchedMul(Wg4, Rfn4)), batchedMul(Wu4, Rfn4)));
+Wra5 = MatrixWithName("rmsn_301.weight", D, 1);
+Xna5 = rmsNorm(X5, Wra5, 1e-5);
+Wq5 = MatrixWithName("gemm_35.weight", Dq, D, 1, 1);
+Qh5 = permute(reshape(batchedMul(Wq5, Xna5), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn5 = MatrixWithName("rmsn_302.weight", hd, 1);
+Qn5 = rmsNorm(Qh5, Wqn5, 1e-5);
+Qr5 = rope(Qn5, cos_te, sin_te);
+Wk5 = MatrixWithName("gemm_36.weight", Dkv, D, 1, 1);
+Kh5 = permute(reshape(batchedMul(Wk5, Xna5), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn5 = MatrixWithName("rmsn_303.weight", hd, 1);
+Kn5 = rmsNorm(Kh5, Wkn5, 1e-5);
+Kr5 = rope(Kn5, cos_te, sin_te);
+Wv5 = MatrixWithName("gemm_37.weight", Dkv, D, 1, 1);
+V5 = permute(reshape(batchedMul(Wv5, Xna5), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc5 = reshapeBatch(Kr5, {hd, T, HkvB, 1});
+Krt5 = tile(Krc5, {1, 1, 1, HkvMul});
+Krp5 = permute(Krt5, {0, 1, 3, 2});
+Kt5 = reshapeBatch(Krp5, {hd, T, 1, HB});
+Vrc5 = reshapeBatch(V5, {hd, T, HkvB, 1});
+Vrt5 = tile(Vrc5, {1, 1, 1, HkvMul});
+Vrp5 = permute(Vrt5, {0, 1, 3, 2});
+Vt5 = reshapeBatch(Vrp5, {hd, T, 1, HB});
+A5 = attention(Qr5, Kt5, Vt5, hd, 1);
+Wo5 = MatrixWithName("gemm_38.weight", D, Dq, 1, 1);
+O5 = batchedMul(Wo5, reshape(permute(A5, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R5 = X5 + O5;
+Wrf5 = MatrixWithName("rmsn_304.weight", D, 1);
+Rfn5 = rmsNorm(R5, Wrf5, 1e-5);
+Wg5 = MatrixWithName("gemm_39.weight", I, D, 1, 1);
+Wu5 = MatrixWithName("gemm_40.weight", I, D, 1, 1);
+Wd5 = MatrixWithName("gemm_41.weight", D, I, 1, 1);
+X6 = R5 + batchedMul(Wd5, elementMul(silu(batchedMul(Wg5, Rfn5)), batchedMul(Wu5, Rfn5)));
+Wra6 = MatrixWithName("rmsn_305.weight", D, 1);
+Xna6 = rmsNorm(X6, Wra6, 1e-5);
+Wq6 = MatrixWithName("gemm_42.weight", Dq, D, 1, 1);
+Qh6 = permute(reshape(batchedMul(Wq6, Xna6), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn6 = MatrixWithName("rmsn_306.weight", hd, 1);
+Qn6 = rmsNorm(Qh6, Wqn6, 1e-5);
+Qr6 = rope(Qn6, cos_te, sin_te);
+Wk6 = MatrixWithName("gemm_43.weight", Dkv, D, 1, 1);
+Kh6 = permute(reshape(batchedMul(Wk6, Xna6), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn6 = MatrixWithName("rmsn_307.weight", hd, 1);
+Kn6 = rmsNorm(Kh6, Wkn6, 1e-5);
+Kr6 = rope(Kn6, cos_te, sin_te);
+Wv6 = MatrixWithName("gemm_44.weight", Dkv, D, 1, 1);
+V6 = permute(reshape(batchedMul(Wv6, Xna6), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc6 = reshapeBatch(Kr6, {hd, T, HkvB, 1});
+Krt6 = tile(Krc6, {1, 1, 1, HkvMul});
+Krp6 = permute(Krt6, {0, 1, 3, 2});
+Kt6 = reshapeBatch(Krp6, {hd, T, 1, HB});
+Vrc6 = reshapeBatch(V6, {hd, T, HkvB, 1});
+Vrt6 = tile(Vrc6, {1, 1, 1, HkvMul});
+Vrp6 = permute(Vrt6, {0, 1, 3, 2});
+Vt6 = reshapeBatch(Vrp6, {hd, T, 1, HB});
+A6 = attention(Qr6, Kt6, Vt6, hd, 1);
+Wo6 = MatrixWithName("gemm_45.weight", D, Dq, 1, 1);
+O6 = batchedMul(Wo6, reshape(permute(A6, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R6 = X6 + O6;
+Wrf6 = MatrixWithName("rmsn_308.weight", D, 1);
+Rfn6 = rmsNorm(R6, Wrf6, 1e-5);
+Wg6 = MatrixWithName("gemm_46.weight", I, D, 1, 1);
+Wu6 = MatrixWithName("gemm_47.weight", I, D, 1, 1);
+Wd6 = MatrixWithName("gemm_48.weight", D, I, 1, 1);
+X7 = R6 + batchedMul(Wd6, elementMul(silu(batchedMul(Wg6, Rfn6)), batchedMul(Wu6, Rfn6)));
+Wra7 = MatrixWithName("rmsn_309.weight", D, 1);
+Xna7 = rmsNorm(X7, Wra7, 1e-5);
+Wq7 = MatrixWithName("gemm_49.weight", Dq, D, 1, 1);
+Qh7 = permute(reshape(batchedMul(Wq7, Xna7), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn7 = MatrixWithName("rmsn_310.weight", hd, 1);
+Qn7 = rmsNorm(Qh7, Wqn7, 1e-5);
+Qr7 = rope(Qn7, cos_te, sin_te);
+Wk7 = MatrixWithName("gemm_50.weight", Dkv, D, 1, 1);
+Kh7 = permute(reshape(batchedMul(Wk7, Xna7), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn7 = MatrixWithName("rmsn_311.weight", hd, 1);
+Kn7 = rmsNorm(Kh7, Wkn7, 1e-5);
+Kr7 = rope(Kn7, cos_te, sin_te);
+Wv7 = MatrixWithName("gemm_51.weight", Dkv, D, 1, 1);
+V7 = permute(reshape(batchedMul(Wv7, Xna7), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc7 = reshapeBatch(Kr7, {hd, T, HkvB, 1});
+Krt7 = tile(Krc7, {1, 1, 1, HkvMul});
+Krp7 = permute(Krt7, {0, 1, 3, 2});
+Kt7 = reshapeBatch(Krp7, {hd, T, 1, HB});
+Vrc7 = reshapeBatch(V7, {hd, T, HkvB, 1});
+Vrt7 = tile(Vrc7, {1, 1, 1, HkvMul});
+Vrp7 = permute(Vrt7, {0, 1, 3, 2});
+Vt7 = reshapeBatch(Vrp7, {hd, T, 1, HB});
+A7 = attention(Qr7, Kt7, Vt7, hd, 1);
+Wo7 = MatrixWithName("gemm_52.weight", D, Dq, 1, 1);
+O7 = batchedMul(Wo7, reshape(permute(A7, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R7 = X7 + O7;
+Wrf7 = MatrixWithName("rmsn_312.weight", D, 1);
+Rfn7 = rmsNorm(R7, Wrf7, 1e-5);
+Wg7 = MatrixWithName("gemm_53.weight", I, D, 1, 1);
+Wu7 = MatrixWithName("gemm_54.weight", I, D, 1, 1);
+Wd7 = MatrixWithName("gemm_55.weight", D, I, 1, 1);
+X8 = R7 + batchedMul(Wd7, elementMul(silu(batchedMul(Wg7, Rfn7)), batchedMul(Wu7, Rfn7)));
+Wra8 = MatrixWithName("rmsn_313.weight", D, 1);
+Xna8 = rmsNorm(X8, Wra8, 1e-5);
+Wq8 = MatrixWithName("gemm_56.weight", Dq, D, 1, 1);
+Qh8 = permute(reshape(batchedMul(Wq8, Xna8), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn8 = MatrixWithName("rmsn_314.weight", hd, 1);
+Qn8 = rmsNorm(Qh8, Wqn8, 1e-5);
+Qr8 = rope(Qn8, cos_te, sin_te);
+Wk8 = MatrixWithName("gemm_57.weight", Dkv, D, 1, 1);
+Kh8 = permute(reshape(batchedMul(Wk8, Xna8), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn8 = MatrixWithName("rmsn_315.weight", hd, 1);
+Kn8 = rmsNorm(Kh8, Wkn8, 1e-5);
+Kr8 = rope(Kn8, cos_te, sin_te);
+Wv8 = MatrixWithName("gemm_58.weight", Dkv, D, 1, 1);
+V8 = permute(reshape(batchedMul(Wv8, Xna8), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc8 = reshapeBatch(Kr8, {hd, T, HkvB, 1});
+Krt8 = tile(Krc8, {1, 1, 1, HkvMul});
+Krp8 = permute(Krt8, {0, 1, 3, 2});
+Kt8 = reshapeBatch(Krp8, {hd, T, 1, HB});
+Vrc8 = reshapeBatch(V8, {hd, T, HkvB, 1});
+Vrt8 = tile(Vrc8, {1, 1, 1, HkvMul});
+Vrp8 = permute(Vrt8, {0, 1, 3, 2});
+Vt8 = reshapeBatch(Vrp8, {hd, T, 1, HB});
+A8 = attention(Qr8, Kt8, Vt8, hd, 1);
+Wo8 = MatrixWithName("gemm_59.weight", D, Dq, 1, 1);
+O8 = batchedMul(Wo8, reshape(permute(A8, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R8 = X8 + O8;
+Wrf8 = MatrixWithName("rmsn_316.weight", D, 1);
+Rfn8 = rmsNorm(R8, Wrf8, 1e-5);
+Wg8 = MatrixWithName("gemm_60.weight", I, D, 1, 1);
+Wu8 = MatrixWithName("gemm_61.weight", I, D, 1, 1);
+Wd8 = MatrixWithName("gemm_62.weight", D, I, 1, 1);
+X9 = R8 + batchedMul(Wd8, elementMul(silu(batchedMul(Wg8, Rfn8)), batchedMul(Wu8, Rfn8)));
+Wra9 = MatrixWithName("rmsn_317.weight", D, 1);
+Xna9 = rmsNorm(X9, Wra9, 1e-5);
+Wq9 = MatrixWithName("gemm_63.weight", Dq, D, 1, 1);
+Qh9 = permute(reshape(batchedMul(Wq9, Xna9), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn9 = MatrixWithName("rmsn_318.weight", hd, 1);
+Qn9 = rmsNorm(Qh9, Wqn9, 1e-5);
+Qr9 = rope(Qn9, cos_te, sin_te);
+Wk9 = MatrixWithName("gemm_64.weight", Dkv, D, 1, 1);
+Kh9 = permute(reshape(batchedMul(Wk9, Xna9), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn9 = MatrixWithName("rmsn_319.weight", hd, 1);
+Kn9 = rmsNorm(Kh9, Wkn9, 1e-5);
+Kr9 = rope(Kn9, cos_te, sin_te);
+Wv9 = MatrixWithName("gemm_65.weight", Dkv, D, 1, 1);
+V9 = permute(reshape(batchedMul(Wv9, Xna9), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc9 = reshapeBatch(Kr9, {hd, T, HkvB, 1});
+Krt9 = tile(Krc9, {1, 1, 1, HkvMul});
+Krp9 = permute(Krt9, {0, 1, 3, 2});
+Kt9 = reshapeBatch(Krp9, {hd, T, 1, HB});
+Vrc9 = reshapeBatch(V9, {hd, T, HkvB, 1});
+Vrt9 = tile(Vrc9, {1, 1, 1, HkvMul});
+Vrp9 = permute(Vrt9, {0, 1, 3, 2});
+Vt9 = reshapeBatch(Vrp9, {hd, T, 1, HB});
+A9 = attention(Qr9, Kt9, Vt9, hd, 1);
+Wo9 = MatrixWithName("gemm_66.weight", D, Dq, 1, 1);
+O9 = batchedMul(Wo9, reshape(permute(A9, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R9 = X9 + O9;
+Wrf9 = MatrixWithName("rmsn_320.weight", D, 1);
+Rfn9 = rmsNorm(R9, Wrf9, 1e-5);
+Wg9 = MatrixWithName("gemm_67.weight", I, D, 1, 1);
+Wu9 = MatrixWithName("gemm_68.weight", I, D, 1, 1);
+Wd9 = MatrixWithName("gemm_69.weight", D, I, 1, 1);
+X10 = R9 + batchedMul(Wd9, elementMul(silu(batchedMul(Wg9, Rfn9)), batchedMul(Wu9, Rfn9)));
+Wra10 = MatrixWithName("rmsn_321.weight", D, 1);
+Xna10 = rmsNorm(X10, Wra10, 1e-5);
+Wq10 = MatrixWithName("gemm_70.weight", Dq, D, 1, 1);
+Qh10 = permute(reshape(batchedMul(Wq10, Xna10), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn10 = MatrixWithName("rmsn_322.weight", hd, 1);
+Qn10 = rmsNorm(Qh10, Wqn10, 1e-5);
+Qr10 = rope(Qn10, cos_te, sin_te);
+Wk10 = MatrixWithName("gemm_71.weight", Dkv, D, 1, 1);
+Kh10 = permute(reshape(batchedMul(Wk10, Xna10), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn10 = MatrixWithName("rmsn_323.weight", hd, 1);
+Kn10 = rmsNorm(Kh10, Wkn10, 1e-5);
+Kr10 = rope(Kn10, cos_te, sin_te);
+Wv10 = MatrixWithName("gemm_72.weight", Dkv, D, 1, 1);
+V10 = permute(reshape(batchedMul(Wv10, Xna10), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc10 = reshapeBatch(Kr10, {hd, T, HkvB, 1});
+Krt10 = tile(Krc10, {1, 1, 1, HkvMul});
+Krp10 = permute(Krt10, {0, 1, 3, 2});
+Kt10 = reshapeBatch(Krp10, {hd, T, 1, HB});
+Vrc10 = reshapeBatch(V10, {hd, T, HkvB, 1});
+Vrt10 = tile(Vrc10, {1, 1, 1, HkvMul});
+Vrp10 = permute(Vrt10, {0, 1, 3, 2});
+Vt10 = reshapeBatch(Vrp10, {hd, T, 1, HB});
+A10 = attention(Qr10, Kt10, Vt10, hd, 1);
+Wo10 = MatrixWithName("gemm_73.weight", D, Dq, 1, 1);
+O10 = batchedMul(Wo10, reshape(permute(A10, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R10 = X10 + O10;
+Wrf10 = MatrixWithName("rmsn_324.weight", D, 1);
+Rfn10 = rmsNorm(R10, Wrf10, 1e-5);
+Wg10 = MatrixWithName("gemm_74.weight", I, D, 1, 1);
+Wu10 = MatrixWithName("gemm_75.weight", I, D, 1, 1);
+Wd10 = MatrixWithName("gemm_76.weight", D, I, 1, 1);
+X11 = R10 + batchedMul(Wd10, elementMul(silu(batchedMul(Wg10, Rfn10)), batchedMul(Wu10, Rfn10)));
+Wra11 = MatrixWithName("rmsn_325.weight", D, 1);
+Xna11 = rmsNorm(X11, Wra11, 1e-5);
+Wq11 = MatrixWithName("gemm_77.weight", Dq, D, 1, 1);
+Qh11 = permute(reshape(batchedMul(Wq11, Xna11), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn11 = MatrixWithName("rmsn_326.weight", hd, 1);
+Qn11 = rmsNorm(Qh11, Wqn11, 1e-5);
+Qr11 = rope(Qn11, cos_te, sin_te);
+Wk11 = MatrixWithName("gemm_78.weight", Dkv, D, 1, 1);
+Kh11 = permute(reshape(batchedMul(Wk11, Xna11), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn11 = MatrixWithName("rmsn_327.weight", hd, 1);
+Kn11 = rmsNorm(Kh11, Wkn11, 1e-5);
+Kr11 = rope(Kn11, cos_te, sin_te);
+Wv11 = MatrixWithName("gemm_79.weight", Dkv, D, 1, 1);
+V11 = permute(reshape(batchedMul(Wv11, Xna11), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc11 = reshapeBatch(Kr11, {hd, T, HkvB, 1});
+Krt11 = tile(Krc11, {1, 1, 1, HkvMul});
+Krp11 = permute(Krt11, {0, 1, 3, 2});
+Kt11 = reshapeBatch(Krp11, {hd, T, 1, HB});
+Vrc11 = reshapeBatch(V11, {hd, T, HkvB, 1});
+Vrt11 = tile(Vrc11, {1, 1, 1, HkvMul});
+Vrp11 = permute(Vrt11, {0, 1, 3, 2});
+Vt11 = reshapeBatch(Vrp11, {hd, T, 1, HB});
+A11 = attention(Qr11, Kt11, Vt11, hd, 1);
+Wo11 = MatrixWithName("gemm_80.weight", D, Dq, 1, 1);
+O11 = batchedMul(Wo11, reshape(permute(A11, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R11 = X11 + O11;
+Wrf11 = MatrixWithName("rmsn_328.weight", D, 1);
+Rfn11 = rmsNorm(R11, Wrf11, 1e-5);
+Wg11 = MatrixWithName("gemm_81.weight", I, D, 1, 1);
+Wu11 = MatrixWithName("gemm_82.weight", I, D, 1, 1);
+Wd11 = MatrixWithName("gemm_83.weight", D, I, 1, 1);
+X12 = R11 + batchedMul(Wd11, elementMul(silu(batchedMul(Wg11, Rfn11)), batchedMul(Wu11, Rfn11)));
+Wra12 = MatrixWithName("rmsn_329.weight", D, 1);
+Xna12 = rmsNorm(X12, Wra12, 1e-5);
+Wq12 = MatrixWithName("gemm_84.weight", Dq, D, 1, 1);
+Qh12 = permute(reshape(batchedMul(Wq12, Xna12), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn12 = MatrixWithName("rmsn_330.weight", hd, 1);
+Qn12 = rmsNorm(Qh12, Wqn12, 1e-5);
+Qr12 = rope(Qn12, cos_te, sin_te);
+Wk12 = MatrixWithName("gemm_85.weight", Dkv, D, 1, 1);
+Kh12 = permute(reshape(batchedMul(Wk12, Xna12), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn12 = MatrixWithName("rmsn_331.weight", hd, 1);
+Kn12 = rmsNorm(Kh12, Wkn12, 1e-5);
+Kr12 = rope(Kn12, cos_te, sin_te);
+Wv12 = MatrixWithName("gemm_86.weight", Dkv, D, 1, 1);
+V12 = permute(reshape(batchedMul(Wv12, Xna12), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc12 = reshapeBatch(Kr12, {hd, T, HkvB, 1});
+Krt12 = tile(Krc12, {1, 1, 1, HkvMul});
+Krp12 = permute(Krt12, {0, 1, 3, 2});
+Kt12 = reshapeBatch(Krp12, {hd, T, 1, HB});
+Vrc12 = reshapeBatch(V12, {hd, T, HkvB, 1});
+Vrt12 = tile(Vrc12, {1, 1, 1, HkvMul});
+Vrp12 = permute(Vrt12, {0, 1, 3, 2});
+Vt12 = reshapeBatch(Vrp12, {hd, T, 1, HB});
+A12 = attention(Qr12, Kt12, Vt12, hd, 1);
+Wo12 = MatrixWithName("gemm_87.weight", D, Dq, 1, 1);
+O12 = batchedMul(Wo12, reshape(permute(A12, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R12 = X12 + O12;
+Wrf12 = MatrixWithName("rmsn_332.weight", D, 1);
+Rfn12 = rmsNorm(R12, Wrf12, 1e-5);
+Wg12 = MatrixWithName("gemm_88.weight", I, D, 1, 1);
+Wu12 = MatrixWithName("gemm_89.weight", I, D, 1, 1);
+Wd12 = MatrixWithName("gemm_90.weight", D, I, 1, 1);
+X13 = R12 + batchedMul(Wd12, elementMul(silu(batchedMul(Wg12, Rfn12)), batchedMul(Wu12, Rfn12)));
+Wra13 = MatrixWithName("rmsn_333.weight", D, 1);
+Xna13 = rmsNorm(X13, Wra13, 1e-5);
+Wq13 = MatrixWithName("gemm_91.weight", Dq, D, 1, 1);
+Qh13 = permute(reshape(batchedMul(Wq13, Xna13), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn13 = MatrixWithName("rmsn_334.weight", hd, 1);
+Qn13 = rmsNorm(Qh13, Wqn13, 1e-5);
+Qr13 = rope(Qn13, cos_te, sin_te);
+Wk13 = MatrixWithName("gemm_92.weight", Dkv, D, 1, 1);
+Kh13 = permute(reshape(batchedMul(Wk13, Xna13), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn13 = MatrixWithName("rmsn_335.weight", hd, 1);
+Kn13 = rmsNorm(Kh13, Wkn13, 1e-5);
+Kr13 = rope(Kn13, cos_te, sin_te);
+Wv13 = MatrixWithName("gemm_93.weight", Dkv, D, 1, 1);
+V13 = permute(reshape(batchedMul(Wv13, Xna13), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc13 = reshapeBatch(Kr13, {hd, T, HkvB, 1});
+Krt13 = tile(Krc13, {1, 1, 1, HkvMul});
+Krp13 = permute(Krt13, {0, 1, 3, 2});
+Kt13 = reshapeBatch(Krp13, {hd, T, 1, HB});
+Vrc13 = reshapeBatch(V13, {hd, T, HkvB, 1});
+Vrt13 = tile(Vrc13, {1, 1, 1, HkvMul});
+Vrp13 = permute(Vrt13, {0, 1, 3, 2});
+Vt13 = reshapeBatch(Vrp13, {hd, T, 1, HB});
+A13 = attention(Qr13, Kt13, Vt13, hd, 1);
+Wo13 = MatrixWithName("gemm_94.weight", D, Dq, 1, 1);
+O13 = batchedMul(Wo13, reshape(permute(A13, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R13 = X13 + O13;
+Wrf13 = MatrixWithName("rmsn_336.weight", D, 1);
+Rfn13 = rmsNorm(R13, Wrf13, 1e-5);
+Wg13 = MatrixWithName("gemm_95.weight", I, D, 1, 1);
+Wu13 = MatrixWithName("gemm_96.weight", I, D, 1, 1);
+Wd13 = MatrixWithName("gemm_97.weight", D, I, 1, 1);
+X14 = R13 + batchedMul(Wd13, elementMul(silu(batchedMul(Wg13, Rfn13)), batchedMul(Wu13, Rfn13)));
+Wra14 = MatrixWithName("rmsn_337.weight", D, 1);
+Xna14 = rmsNorm(X14, Wra14, 1e-5);
+Wq14 = MatrixWithName("gemm_98.weight", Dq, D, 1, 1);
+Qh14 = permute(reshape(batchedMul(Wq14, Xna14), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn14 = MatrixWithName("rmsn_338.weight", hd, 1);
+Qn14 = rmsNorm(Qh14, Wqn14, 1e-5);
+Qr14 = rope(Qn14, cos_te, sin_te);
+Wk14 = MatrixWithName("gemm_99.weight", Dkv, D, 1, 1);
+Kh14 = permute(reshape(batchedMul(Wk14, Xna14), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn14 = MatrixWithName("rmsn_339.weight", hd, 1);
+Kn14 = rmsNorm(Kh14, Wkn14, 1e-5);
+Kr14 = rope(Kn14, cos_te, sin_te);
+Wv14 = MatrixWithName("gemm_100.weight", Dkv, D, 1, 1);
+V14 = permute(reshape(batchedMul(Wv14, Xna14), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc14 = reshapeBatch(Kr14, {hd, T, HkvB, 1});
+Krt14 = tile(Krc14, {1, 1, 1, HkvMul});
+Krp14 = permute(Krt14, {0, 1, 3, 2});
+Kt14 = reshapeBatch(Krp14, {hd, T, 1, HB});
+Vrc14 = reshapeBatch(V14, {hd, T, HkvB, 1});
+Vrt14 = tile(Vrc14, {1, 1, 1, HkvMul});
+Vrp14 = permute(Vrt14, {0, 1, 3, 2});
+Vt14 = reshapeBatch(Vrp14, {hd, T, 1, HB});
+A14 = attention(Qr14, Kt14, Vt14, hd, 1);
+Wo14 = MatrixWithName("gemm_101.weight", D, Dq, 1, 1);
+O14 = batchedMul(Wo14, reshape(permute(A14, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R14 = X14 + O14;
+Wrf14 = MatrixWithName("rmsn_340.weight", D, 1);
+Rfn14 = rmsNorm(R14, Wrf14, 1e-5);
+Wg14 = MatrixWithName("gemm_102.weight", I, D, 1, 1);
+Wu14 = MatrixWithName("gemm_103.weight", I, D, 1, 1);
+Wd14 = MatrixWithName("gemm_104.weight", D, I, 1, 1);
+X15 = R14 + batchedMul(Wd14, elementMul(silu(batchedMul(Wg14, Rfn14)), batchedMul(Wu14, Rfn14)));
+Wra15 = MatrixWithName("rmsn_341.weight", D, 1);
+Xna15 = rmsNorm(X15, Wra15, 1e-5);
+Wq15 = MatrixWithName("gemm_105.weight", Dq, D, 1, 1);
+Qh15 = permute(reshape(batchedMul(Wq15, Xna15), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn15 = MatrixWithName("rmsn_342.weight", hd, 1);
+Qn15 = rmsNorm(Qh15, Wqn15, 1e-5);
+Qr15 = rope(Qn15, cos_te, sin_te);
+Wk15 = MatrixWithName("gemm_106.weight", Dkv, D, 1, 1);
+Kh15 = permute(reshape(batchedMul(Wk15, Xna15), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn15 = MatrixWithName("rmsn_343.weight", hd, 1);
+Kn15 = rmsNorm(Kh15, Wkn15, 1e-5);
+Kr15 = rope(Kn15, cos_te, sin_te);
+Wv15 = MatrixWithName("gemm_107.weight", Dkv, D, 1, 1);
+V15 = permute(reshape(batchedMul(Wv15, Xna15), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc15 = reshapeBatch(Kr15, {hd, T, HkvB, 1});
+Krt15 = tile(Krc15, {1, 1, 1, HkvMul});
+Krp15 = permute(Krt15, {0, 1, 3, 2});
+Kt15 = reshapeBatch(Krp15, {hd, T, 1, HB});
+Vrc15 = reshapeBatch(V15, {hd, T, HkvB, 1});
+Vrt15 = tile(Vrc15, {1, 1, 1, HkvMul});
+Vrp15 = permute(Vrt15, {0, 1, 3, 2});
+Vt15 = reshapeBatch(Vrp15, {hd, T, 1, HB});
+A15 = attention(Qr15, Kt15, Vt15, hd, 1);
+Wo15 = MatrixWithName("gemm_108.weight", D, Dq, 1, 1);
+O15 = batchedMul(Wo15, reshape(permute(A15, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R15 = X15 + O15;
+Wrf15 = MatrixWithName("rmsn_344.weight", D, 1);
+Rfn15 = rmsNorm(R15, Wrf15, 1e-5);
+Wg15 = MatrixWithName("gemm_109.weight", I, D, 1, 1);
+Wu15 = MatrixWithName("gemm_110.weight", I, D, 1, 1);
+Wd15 = MatrixWithName("gemm_111.weight", D, I, 1, 1);
+X16 = R15 + batchedMul(Wd15, elementMul(silu(batchedMul(Wg15, Rfn15)), batchedMul(Wu15, Rfn15)));
+Wra16 = MatrixWithName("rmsn_345.weight", D, 1);
+Xna16 = rmsNorm(X16, Wra16, 1e-5);
+Wq16 = MatrixWithName("gemm_112.weight", Dq, D, 1, 1);
+Qh16 = permute(reshape(batchedMul(Wq16, Xna16), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn16 = MatrixWithName("rmsn_346.weight", hd, 1);
+Qn16 = rmsNorm(Qh16, Wqn16, 1e-5);
+Qr16 = rope(Qn16, cos_te, sin_te);
+Wk16 = MatrixWithName("gemm_113.weight", Dkv, D, 1, 1);
+Kh16 = permute(reshape(batchedMul(Wk16, Xna16), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn16 = MatrixWithName("rmsn_347.weight", hd, 1);
+Kn16 = rmsNorm(Kh16, Wkn16, 1e-5);
+Kr16 = rope(Kn16, cos_te, sin_te);
+Wv16 = MatrixWithName("gemm_114.weight", Dkv, D, 1, 1);
+V16 = permute(reshape(batchedMul(Wv16, Xna16), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc16 = reshapeBatch(Kr16, {hd, T, HkvB, 1});
+Krt16 = tile(Krc16, {1, 1, 1, HkvMul});
+Krp16 = permute(Krt16, {0, 1, 3, 2});
+Kt16 = reshapeBatch(Krp16, {hd, T, 1, HB});
+Vrc16 = reshapeBatch(V16, {hd, T, HkvB, 1});
+Vrt16 = tile(Vrc16, {1, 1, 1, HkvMul});
+Vrp16 = permute(Vrt16, {0, 1, 3, 2});
+Vt16 = reshapeBatch(Vrp16, {hd, T, 1, HB});
+A16 = attention(Qr16, Kt16, Vt16, hd, 1);
+Wo16 = MatrixWithName("gemm_115.weight", D, Dq, 1, 1);
+O16 = batchedMul(Wo16, reshape(permute(A16, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R16 = X16 + O16;
+Wrf16 = MatrixWithName("rmsn_348.weight", D, 1);
+Rfn16 = rmsNorm(R16, Wrf16, 1e-5);
+Wg16 = MatrixWithName("gemm_116.weight", I, D, 1, 1);
+Wu16 = MatrixWithName("gemm_117.weight", I, D, 1, 1);
+Wd16 = MatrixWithName("gemm_118.weight", D, I, 1, 1);
+X17 = R16 + batchedMul(Wd16, elementMul(silu(batchedMul(Wg16, Rfn16)), batchedMul(Wu16, Rfn16)));
+Wra17 = MatrixWithName("rmsn_349.weight", D, 1);
+Xna17 = rmsNorm(X17, Wra17, 1e-5);
+Wq17 = MatrixWithName("gemm_119.weight", Dq, D, 1, 1);
+Qh17 = permute(reshape(batchedMul(Wq17, Xna17), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn17 = MatrixWithName("rmsn_350.weight", hd, 1);
+Qn17 = rmsNorm(Qh17, Wqn17, 1e-5);
+Qr17 = rope(Qn17, cos_te, sin_te);
+Wk17 = MatrixWithName("gemm_120.weight", Dkv, D, 1, 1);
+Kh17 = permute(reshape(batchedMul(Wk17, Xna17), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn17 = MatrixWithName("rmsn_351.weight", hd, 1);
+Kn17 = rmsNorm(Kh17, Wkn17, 1e-5);
+Kr17 = rope(Kn17, cos_te, sin_te);
+Wv17 = MatrixWithName("gemm_121.weight", Dkv, D, 1, 1);
+V17 = permute(reshape(batchedMul(Wv17, Xna17), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc17 = reshapeBatch(Kr17, {hd, T, HkvB, 1});
+Krt17 = tile(Krc17, {1, 1, 1, HkvMul});
+Krp17 = permute(Krt17, {0, 1, 3, 2});
+Kt17 = reshapeBatch(Krp17, {hd, T, 1, HB});
+Vrc17 = reshapeBatch(V17, {hd, T, HkvB, 1});
+Vrt17 = tile(Vrc17, {1, 1, 1, HkvMul});
+Vrp17 = permute(Vrt17, {0, 1, 3, 2});
+Vt17 = reshapeBatch(Vrp17, {hd, T, 1, HB});
+A17 = attention(Qr17, Kt17, Vt17, hd, 1);
+Wo17 = MatrixWithName("gemm_122.weight", D, Dq, 1, 1);
+O17 = batchedMul(Wo17, reshape(permute(A17, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R17 = X17 + O17;
+Wrf17 = MatrixWithName("rmsn_352.weight", D, 1);
+Rfn17 = rmsNorm(R17, Wrf17, 1e-5);
+Wg17 = MatrixWithName("gemm_123.weight", I, D, 1, 1);
+Wu17 = MatrixWithName("gemm_124.weight", I, D, 1, 1);
+Wd17 = MatrixWithName("gemm_125.weight", D, I, 1, 1);
+X18 = R17 + batchedMul(Wd17, elementMul(silu(batchedMul(Wg17, Rfn17)), batchedMul(Wu17, Rfn17)));
+Wra18 = MatrixWithName("rmsn_353.weight", D, 1);
+Xna18 = rmsNorm(X18, Wra18, 1e-5);
+Wq18 = MatrixWithName("gemm_126.weight", Dq, D, 1, 1);
+Qh18 = permute(reshape(batchedMul(Wq18, Xna18), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn18 = MatrixWithName("rmsn_354.weight", hd, 1);
+Qn18 = rmsNorm(Qh18, Wqn18, 1e-5);
+Qr18 = rope(Qn18, cos_te, sin_te);
+Wk18 = MatrixWithName("gemm_127.weight", Dkv, D, 1, 1);
+Kh18 = permute(reshape(batchedMul(Wk18, Xna18), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn18 = MatrixWithName("rmsn_355.weight", hd, 1);
+Kn18 = rmsNorm(Kh18, Wkn18, 1e-5);
+Kr18 = rope(Kn18, cos_te, sin_te);
+Wv18 = MatrixWithName("gemm_128.weight", Dkv, D, 1, 1);
+V18 = permute(reshape(batchedMul(Wv18, Xna18), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc18 = reshapeBatch(Kr18, {hd, T, HkvB, 1});
+Krt18 = tile(Krc18, {1, 1, 1, HkvMul});
+Krp18 = permute(Krt18, {0, 1, 3, 2});
+Kt18 = reshapeBatch(Krp18, {hd, T, 1, HB});
+Vrc18 = reshapeBatch(V18, {hd, T, HkvB, 1});
+Vrt18 = tile(Vrc18, {1, 1, 1, HkvMul});
+Vrp18 = permute(Vrt18, {0, 1, 3, 2});
+Vt18 = reshapeBatch(Vrp18, {hd, T, 1, HB});
+A18 = attention(Qr18, Kt18, Vt18, hd, 1);
+Wo18 = MatrixWithName("gemm_129.weight", D, Dq, 1, 1);
+O18 = batchedMul(Wo18, reshape(permute(A18, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R18 = X18 + O18;
+Wrf18 = MatrixWithName("rmsn_356.weight", D, 1);
+Rfn18 = rmsNorm(R18, Wrf18, 1e-5);
+Wg18 = MatrixWithName("gemm_130.weight", I, D, 1, 1);
+Wu18 = MatrixWithName("gemm_131.weight", I, D, 1, 1);
+Wd18 = MatrixWithName("gemm_132.weight", D, I, 1, 1);
+X19 = R18 + batchedMul(Wd18, elementMul(silu(batchedMul(Wg18, Rfn18)), batchedMul(Wu18, Rfn18)));
+Wra19 = MatrixWithName("rmsn_357.weight", D, 1);
+Xna19 = rmsNorm(X19, Wra19, 1e-5);
+Wq19 = MatrixWithName("gemm_133.weight", Dq, D, 1, 1);
+Qh19 = permute(reshape(batchedMul(Wq19, Xna19), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn19 = MatrixWithName("rmsn_358.weight", hd, 1);
+Qn19 = rmsNorm(Qh19, Wqn19, 1e-5);
+Qr19 = rope(Qn19, cos_te, sin_te);
+Wk19 = MatrixWithName("gemm_134.weight", Dkv, D, 1, 1);
+Kh19 = permute(reshape(batchedMul(Wk19, Xna19), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn19 = MatrixWithName("rmsn_359.weight", hd, 1);
+Kn19 = rmsNorm(Kh19, Wkn19, 1e-5);
+Kr19 = rope(Kn19, cos_te, sin_te);
+Wv19 = MatrixWithName("gemm_135.weight", Dkv, D, 1, 1);
+V19 = permute(reshape(batchedMul(Wv19, Xna19), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc19 = reshapeBatch(Kr19, {hd, T, HkvB, 1});
+Krt19 = tile(Krc19, {1, 1, 1, HkvMul});
+Krp19 = permute(Krt19, {0, 1, 3, 2});
+Kt19 = reshapeBatch(Krp19, {hd, T, 1, HB});
+Vrc19 = reshapeBatch(V19, {hd, T, HkvB, 1});
+Vrt19 = tile(Vrc19, {1, 1, 1, HkvMul});
+Vrp19 = permute(Vrt19, {0, 1, 3, 2});
+Vt19 = reshapeBatch(Vrp19, {hd, T, 1, HB});
+A19 = attention(Qr19, Kt19, Vt19, hd, 1);
+Wo19 = MatrixWithName("gemm_136.weight", D, Dq, 1, 1);
+O19 = batchedMul(Wo19, reshape(permute(A19, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R19 = X19 + O19;
+Wrf19 = MatrixWithName("rmsn_360.weight", D, 1);
+Rfn19 = rmsNorm(R19, Wrf19, 1e-5);
+Wg19 = MatrixWithName("gemm_137.weight", I, D, 1, 1);
+Wu19 = MatrixWithName("gemm_138.weight", I, D, 1, 1);
+Wd19 = MatrixWithName("gemm_139.weight", D, I, 1, 1);
+X20 = R19 + batchedMul(Wd19, elementMul(silu(batchedMul(Wg19, Rfn19)), batchedMul(Wu19, Rfn19)));
+Wra20 = MatrixWithName("rmsn_361.weight", D, 1);
+Xna20 = rmsNorm(X20, Wra20, 1e-5);
+Wq20 = MatrixWithName("gemm_140.weight", Dq, D, 1, 1);
+Qh20 = permute(reshape(batchedMul(Wq20, Xna20), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn20 = MatrixWithName("rmsn_362.weight", hd, 1);
+Qn20 = rmsNorm(Qh20, Wqn20, 1e-5);
+Qr20 = rope(Qn20, cos_te, sin_te);
+Wk20 = MatrixWithName("gemm_141.weight", Dkv, D, 1, 1);
+Kh20 = permute(reshape(batchedMul(Wk20, Xna20), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn20 = MatrixWithName("rmsn_363.weight", hd, 1);
+Kn20 = rmsNorm(Kh20, Wkn20, 1e-5);
+Kr20 = rope(Kn20, cos_te, sin_te);
+Wv20 = MatrixWithName("gemm_142.weight", Dkv, D, 1, 1);
+V20 = permute(reshape(batchedMul(Wv20, Xna20), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc20 = reshapeBatch(Kr20, {hd, T, HkvB, 1});
+Krt20 = tile(Krc20, {1, 1, 1, HkvMul});
+Krp20 = permute(Krt20, {0, 1, 3, 2});
+Kt20 = reshapeBatch(Krp20, {hd, T, 1, HB});
+Vrc20 = reshapeBatch(V20, {hd, T, HkvB, 1});
+Vrt20 = tile(Vrc20, {1, 1, 1, HkvMul});
+Vrp20 = permute(Vrt20, {0, 1, 3, 2});
+Vt20 = reshapeBatch(Vrp20, {hd, T, 1, HB});
+A20 = attention(Qr20, Kt20, Vt20, hd, 1);
+Wo20 = MatrixWithName("gemm_143.weight", D, Dq, 1, 1);
+O20 = batchedMul(Wo20, reshape(permute(A20, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R20 = X20 + O20;
+Wrf20 = MatrixWithName("rmsn_364.weight", D, 1);
+Rfn20 = rmsNorm(R20, Wrf20, 1e-5);
+Wg20 = MatrixWithName("gemm_144.weight", I, D, 1, 1);
+Wu20 = MatrixWithName("gemm_145.weight", I, D, 1, 1);
+Wd20 = MatrixWithName("gemm_146.weight", D, I, 1, 1);
+X21 = R20 + batchedMul(Wd20, elementMul(silu(batchedMul(Wg20, Rfn20)), batchedMul(Wu20, Rfn20)));
+Wra21 = MatrixWithName("rmsn_365.weight", D, 1);
+Xna21 = rmsNorm(X21, Wra21, 1e-5);
+Wq21 = MatrixWithName("gemm_147.weight", Dq, D, 1, 1);
+Qh21 = permute(reshape(batchedMul(Wq21, Xna21), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn21 = MatrixWithName("rmsn_366.weight", hd, 1);
+Qn21 = rmsNorm(Qh21, Wqn21, 1e-5);
+Qr21 = rope(Qn21, cos_te, sin_te);
+Wk21 = MatrixWithName("gemm_148.weight", Dkv, D, 1, 1);
+Kh21 = permute(reshape(batchedMul(Wk21, Xna21), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn21 = MatrixWithName("rmsn_367.weight", hd, 1);
+Kn21 = rmsNorm(Kh21, Wkn21, 1e-5);
+Kr21 = rope(Kn21, cos_te, sin_te);
+Wv21 = MatrixWithName("gemm_149.weight", Dkv, D, 1, 1);
+V21 = permute(reshape(batchedMul(Wv21, Xna21), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc21 = reshapeBatch(Kr21, {hd, T, HkvB, 1});
+Krt21 = tile(Krc21, {1, 1, 1, HkvMul});
+Krp21 = permute(Krt21, {0, 1, 3, 2});
+Kt21 = reshapeBatch(Krp21, {hd, T, 1, HB});
+Vrc21 = reshapeBatch(V21, {hd, T, HkvB, 1});
+Vrt21 = tile(Vrc21, {1, 1, 1, HkvMul});
+Vrp21 = permute(Vrt21, {0, 1, 3, 2});
+Vt21 = reshapeBatch(Vrp21, {hd, T, 1, HB});
+A21 = attention(Qr21, Kt21, Vt21, hd, 1);
+Wo21 = MatrixWithName("gemm_150.weight", D, Dq, 1, 1);
+O21 = batchedMul(Wo21, reshape(permute(A21, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R21 = X21 + O21;
+Wrf21 = MatrixWithName("rmsn_368.weight", D, 1);
+Rfn21 = rmsNorm(R21, Wrf21, 1e-5);
+Wg21 = MatrixWithName("gemm_151.weight", I, D, 1, 1);
+Wu21 = MatrixWithName("gemm_152.weight", I, D, 1, 1);
+Wd21 = MatrixWithName("gemm_153.weight", D, I, 1, 1);
+X22 = R21 + batchedMul(Wd21, elementMul(silu(batchedMul(Wg21, Rfn21)), batchedMul(Wu21, Rfn21)));
+Wra22 = MatrixWithName("rmsn_369.weight", D, 1);
+Xna22 = rmsNorm(X22, Wra22, 1e-5);
+Wq22 = MatrixWithName("gemm_154.weight", Dq, D, 1, 1);
+Qh22 = permute(reshape(batchedMul(Wq22, Xna22), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn22 = MatrixWithName("rmsn_370.weight", hd, 1);
+Qn22 = rmsNorm(Qh22, Wqn22, 1e-5);
+Qr22 = rope(Qn22, cos_te, sin_te);
+Wk22 = MatrixWithName("gemm_155.weight", Dkv, D, 1, 1);
+Kh22 = permute(reshape(batchedMul(Wk22, Xna22), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn22 = MatrixWithName("rmsn_371.weight", hd, 1);
+Kn22 = rmsNorm(Kh22, Wkn22, 1e-5);
+Kr22 = rope(Kn22, cos_te, sin_te);
+Wv22 = MatrixWithName("gemm_156.weight", Dkv, D, 1, 1);
+V22 = permute(reshape(batchedMul(Wv22, Xna22), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc22 = reshapeBatch(Kr22, {hd, T, HkvB, 1});
+Krt22 = tile(Krc22, {1, 1, 1, HkvMul});
+Krp22 = permute(Krt22, {0, 1, 3, 2});
+Kt22 = reshapeBatch(Krp22, {hd, T, 1, HB});
+Vrc22 = reshapeBatch(V22, {hd, T, HkvB, 1});
+Vrt22 = tile(Vrc22, {1, 1, 1, HkvMul});
+Vrp22 = permute(Vrt22, {0, 1, 3, 2});
+Vt22 = reshapeBatch(Vrp22, {hd, T, 1, HB});
+A22 = attention(Qr22, Kt22, Vt22, hd, 1);
+Wo22 = MatrixWithName("gemm_157.weight", D, Dq, 1, 1);
+O22 = batchedMul(Wo22, reshape(permute(A22, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R22 = X22 + O22;
+Wrf22 = MatrixWithName("rmsn_372.weight", D, 1);
+Rfn22 = rmsNorm(R22, Wrf22, 1e-5);
+Wg22 = MatrixWithName("gemm_158.weight", I, D, 1, 1);
+Wu22 = MatrixWithName("gemm_159.weight", I, D, 1, 1);
+Wd22 = MatrixWithName("gemm_160.weight", D, I, 1, 1);
+X23 = R22 + batchedMul(Wd22, elementMul(silu(batchedMul(Wg22, Rfn22)), batchedMul(Wu22, Rfn22)));
+Wra23 = MatrixWithName("rmsn_373.weight", D, 1);
+Xna23 = rmsNorm(X23, Wra23, 1e-5);
+Wq23 = MatrixWithName("gemm_161.weight", Dq, D, 1, 1);
+Qh23 = permute(reshape(batchedMul(Wq23, Xna23), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn23 = MatrixWithName("rmsn_374.weight", hd, 1);
+Qn23 = rmsNorm(Qh23, Wqn23, 1e-5);
+Qr23 = rope(Qn23, cos_te, sin_te);
+Wk23 = MatrixWithName("gemm_162.weight", Dkv, D, 1, 1);
+Kh23 = permute(reshape(batchedMul(Wk23, Xna23), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn23 = MatrixWithName("rmsn_375.weight", hd, 1);
+Kn23 = rmsNorm(Kh23, Wkn23, 1e-5);
+Kr23 = rope(Kn23, cos_te, sin_te);
+Wv23 = MatrixWithName("gemm_163.weight", Dkv, D, 1, 1);
+V23 = permute(reshape(batchedMul(Wv23, Xna23), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc23 = reshapeBatch(Kr23, {hd, T, HkvB, 1});
+Krt23 = tile(Krc23, {1, 1, 1, HkvMul});
+Krp23 = permute(Krt23, {0, 1, 3, 2});
+Kt23 = reshapeBatch(Krp23, {hd, T, 1, HB});
+Vrc23 = reshapeBatch(V23, {hd, T, HkvB, 1});
+Vrt23 = tile(Vrc23, {1, 1, 1, HkvMul});
+Vrp23 = permute(Vrt23, {0, 1, 3, 2});
+Vt23 = reshapeBatch(Vrp23, {hd, T, 1, HB});
+A23 = attention(Qr23, Kt23, Vt23, hd, 1);
+Wo23 = MatrixWithName("gemm_164.weight", D, Dq, 1, 1);
+O23 = batchedMul(Wo23, reshape(permute(A23, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R23 = X23 + O23;
+Wrf23 = MatrixWithName("rmsn_376.weight", D, 1);
+Rfn23 = rmsNorm(R23, Wrf23, 1e-5);
+Wg23 = MatrixWithName("gemm_165.weight", I, D, 1, 1);
+Wu23 = MatrixWithName("gemm_166.weight", I, D, 1, 1);
+Wd23 = MatrixWithName("gemm_167.weight", D, I, 1, 1);
+X24 = R23 + batchedMul(Wd23, elementMul(silu(batchedMul(Wg23, Rfn23)), batchedMul(Wu23, Rfn23)));
+Wra24 = MatrixWithName("rmsn_377.weight", D, 1);
+Xna24 = rmsNorm(X24, Wra24, 1e-5);
+Wq24 = MatrixWithName("gemm_168.weight", Dq, D, 1, 1);
+Qh24 = permute(reshape(batchedMul(Wq24, Xna24), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn24 = MatrixWithName("rmsn_378.weight", hd, 1);
+Qn24 = rmsNorm(Qh24, Wqn24, 1e-5);
+Qr24 = rope(Qn24, cos_te, sin_te);
+Wk24 = MatrixWithName("gemm_169.weight", Dkv, D, 1, 1);
+Kh24 = permute(reshape(batchedMul(Wk24, Xna24), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn24 = MatrixWithName("rmsn_379.weight", hd, 1);
+Kn24 = rmsNorm(Kh24, Wkn24, 1e-5);
+Kr24 = rope(Kn24, cos_te, sin_te);
+Wv24 = MatrixWithName("gemm_170.weight", Dkv, D, 1, 1);
+V24 = permute(reshape(batchedMul(Wv24, Xna24), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc24 = reshapeBatch(Kr24, {hd, T, HkvB, 1});
+Krt24 = tile(Krc24, {1, 1, 1, HkvMul});
+Krp24 = permute(Krt24, {0, 1, 3, 2});
+Kt24 = reshapeBatch(Krp24, {hd, T, 1, HB});
+Vrc24 = reshapeBatch(V24, {hd, T, HkvB, 1});
+Vrt24 = tile(Vrc24, {1, 1, 1, HkvMul});
+Vrp24 = permute(Vrt24, {0, 1, 3, 2});
+Vt24 = reshapeBatch(Vrp24, {hd, T, 1, HB});
+A24 = attention(Qr24, Kt24, Vt24, hd, 1);
+Wo24 = MatrixWithName("gemm_171.weight", D, Dq, 1, 1);
+O24 = batchedMul(Wo24, reshape(permute(A24, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R24 = X24 + O24;
+Wrf24 = MatrixWithName("rmsn_380.weight", D, 1);
+Rfn24 = rmsNorm(R24, Wrf24, 1e-5);
+Wg24 = MatrixWithName("gemm_172.weight", I, D, 1, 1);
+Wu24 = MatrixWithName("gemm_173.weight", I, D, 1, 1);
+Wd24 = MatrixWithName("gemm_174.weight", D, I, 1, 1);
+X25 = R24 + batchedMul(Wd24, elementMul(silu(batchedMul(Wg24, Rfn24)), batchedMul(Wu24, Rfn24)));
+Wra25 = MatrixWithName("rmsn_381.weight", D, 1);
+Xna25 = rmsNorm(X25, Wra25, 1e-5);
+Wq25 = MatrixWithName("gemm_175.weight", Dq, D, 1, 1);
+Qh25 = permute(reshape(batchedMul(Wq25, Xna25), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn25 = MatrixWithName("rmsn_382.weight", hd, 1);
+Qn25 = rmsNorm(Qh25, Wqn25, 1e-5);
+Qr25 = rope(Qn25, cos_te, sin_te);
+Wk25 = MatrixWithName("gemm_176.weight", Dkv, D, 1, 1);
+Kh25 = permute(reshape(batchedMul(Wk25, Xna25), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn25 = MatrixWithName("rmsn_383.weight", hd, 1);
+Kn25 = rmsNorm(Kh25, Wkn25, 1e-5);
+Kr25 = rope(Kn25, cos_te, sin_te);
+Wv25 = MatrixWithName("gemm_177.weight", Dkv, D, 1, 1);
+V25 = permute(reshape(batchedMul(Wv25, Xna25), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc25 = reshapeBatch(Kr25, {hd, T, HkvB, 1});
+Krt25 = tile(Krc25, {1, 1, 1, HkvMul});
+Krp25 = permute(Krt25, {0, 1, 3, 2});
+Kt25 = reshapeBatch(Krp25, {hd, T, 1, HB});
+Vrc25 = reshapeBatch(V25, {hd, T, HkvB, 1});
+Vrt25 = tile(Vrc25, {1, 1, 1, HkvMul});
+Vrp25 = permute(Vrt25, {0, 1, 3, 2});
+Vt25 = reshapeBatch(Vrp25, {hd, T, 1, HB});
+A25 = attention(Qr25, Kt25, Vt25, hd, 1);
+Wo25 = MatrixWithName("gemm_178.weight", D, Dq, 1, 1);
+O25 = batchedMul(Wo25, reshape(permute(A25, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R25 = X25 + O25;
+Wrf25 = MatrixWithName("rmsn_384.weight", D, 1);
+Rfn25 = rmsNorm(R25, Wrf25, 1e-5);
+Wg25 = MatrixWithName("gemm_179.weight", I, D, 1, 1);
+Wu25 = MatrixWithName("gemm_180.weight", I, D, 1, 1);
+Wd25 = MatrixWithName("gemm_181.weight", D, I, 1, 1);
+X26 = R25 + batchedMul(Wd25, elementMul(silu(batchedMul(Wg25, Rfn25)), batchedMul(Wu25, Rfn25)));
+Wra26 = MatrixWithName("rmsn_385.weight", D, 1);
+Xna26 = rmsNorm(X26, Wra26, 1e-5);
+Wq26 = MatrixWithName("gemm_182.weight", Dq, D, 1, 1);
+Qh26 = permute(reshape(batchedMul(Wq26, Xna26), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn26 = MatrixWithName("rmsn_386.weight", hd, 1);
+Qn26 = rmsNorm(Qh26, Wqn26, 1e-5);
+Qr26 = rope(Qn26, cos_te, sin_te);
+Wk26 = MatrixWithName("gemm_183.weight", Dkv, D, 1, 1);
+Kh26 = permute(reshape(batchedMul(Wk26, Xna26), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn26 = MatrixWithName("rmsn_387.weight", hd, 1);
+Kn26 = rmsNorm(Kh26, Wkn26, 1e-5);
+Kr26 = rope(Kn26, cos_te, sin_te);
+Wv26 = MatrixWithName("gemm_184.weight", Dkv, D, 1, 1);
+V26 = permute(reshape(batchedMul(Wv26, Xna26), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc26 = reshapeBatch(Kr26, {hd, T, HkvB, 1});
+Krt26 = tile(Krc26, {1, 1, 1, HkvMul});
+Krp26 = permute(Krt26, {0, 1, 3, 2});
+Kt26 = reshapeBatch(Krp26, {hd, T, 1, HB});
+Vrc26 = reshapeBatch(V26, {hd, T, HkvB, 1});
+Vrt26 = tile(Vrc26, {1, 1, 1, HkvMul});
+Vrp26 = permute(Vrt26, {0, 1, 3, 2});
+Vt26 = reshapeBatch(Vrp26, {hd, T, 1, HB});
+A26 = attention(Qr26, Kt26, Vt26, hd, 1);
+Wo26 = MatrixWithName("gemm_185.weight", D, Dq, 1, 1);
+O26 = batchedMul(Wo26, reshape(permute(A26, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R26 = X26 + O26;
+Wrf26 = MatrixWithName("rmsn_388.weight", D, 1);
+Rfn26 = rmsNorm(R26, Wrf26, 1e-5);
+Wg26 = MatrixWithName("gemm_186.weight", I, D, 1, 1);
+Wu26 = MatrixWithName("gemm_187.weight", I, D, 1, 1);
+Wd26 = MatrixWithName("gemm_188.weight", D, I, 1, 1);
+X27 = R26 + batchedMul(Wd26, elementMul(silu(batchedMul(Wg26, Rfn26)), batchedMul(Wu26, Rfn26)));
+Wra27 = MatrixWithName("rmsn_389.weight", D, 1);
+Xna27 = rmsNorm(X27, Wra27, 1e-5);
+Wq27 = MatrixWithName("gemm_189.weight", Dq, D, 1, 1);
+Qh27 = permute(reshape(batchedMul(Wq27, Xna27), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn27 = MatrixWithName("rmsn_390.weight", hd, 1);
+Qn27 = rmsNorm(Qh27, Wqn27, 1e-5);
+Qr27 = rope(Qn27, cos_te, sin_te);
+Wk27 = MatrixWithName("gemm_190.weight", Dkv, D, 1, 1);
+Kh27 = permute(reshape(batchedMul(Wk27, Xna27), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn27 = MatrixWithName("rmsn_391.weight", hd, 1);
+Kn27 = rmsNorm(Kh27, Wkn27, 1e-5);
+Kr27 = rope(Kn27, cos_te, sin_te);
+Wv27 = MatrixWithName("gemm_191.weight", Dkv, D, 1, 1);
+V27 = permute(reshape(batchedMul(Wv27, Xna27), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc27 = reshapeBatch(Kr27, {hd, T, HkvB, 1});
+Krt27 = tile(Krc27, {1, 1, 1, HkvMul});
+Krp27 = permute(Krt27, {0, 1, 3, 2});
+Kt27 = reshapeBatch(Krp27, {hd, T, 1, HB});
+Vrc27 = reshapeBatch(V27, {hd, T, HkvB, 1});
+Vrt27 = tile(Vrc27, {1, 1, 1, HkvMul});
+Vrp27 = permute(Vrt27, {0, 1, 3, 2});
+Vt27 = reshapeBatch(Vrp27, {hd, T, 1, HB});
+A27 = attention(Qr27, Kt27, Vt27, hd, 1);
+Wo27 = MatrixWithName("gemm_192.weight", D, Dq, 1, 1);
+O27 = batchedMul(Wo27, reshape(permute(A27, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R27 = X27 + O27;
+Wrf27 = MatrixWithName("rmsn_392.weight", D, 1);
+Rfn27 = rmsNorm(R27, Wrf27, 1e-5);
+Wg27 = MatrixWithName("gemm_193.weight", I, D, 1, 1);
+Wu27 = MatrixWithName("gemm_194.weight", I, D, 1, 1);
+Wd27 = MatrixWithName("gemm_195.weight", D, I, 1, 1);
+X28 = R27 + batchedMul(Wd27, elementMul(silu(batchedMul(Wg27, Rfn27)), batchedMul(Wu27, Rfn27)));
+Wra28 = MatrixWithName("rmsn_393.weight", D, 1);
+Xna28 = rmsNorm(X28, Wra28, 1e-5);
+Wq28 = MatrixWithName("gemm_196.weight", Dq, D, 1, 1);
+Qh28 = permute(reshape(batchedMul(Wq28, Xna28), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn28 = MatrixWithName("rmsn_394.weight", hd, 1);
+Qn28 = rmsNorm(Qh28, Wqn28, 1e-5);
+Qr28 = rope(Qn28, cos_te, sin_te);
+Wk28 = MatrixWithName("gemm_197.weight", Dkv, D, 1, 1);
+Kh28 = permute(reshape(batchedMul(Wk28, Xna28), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn28 = MatrixWithName("rmsn_395.weight", hd, 1);
+Kn28 = rmsNorm(Kh28, Wkn28, 1e-5);
+Kr28 = rope(Kn28, cos_te, sin_te);
+Wv28 = MatrixWithName("gemm_198.weight", Dkv, D, 1, 1);
+V28 = permute(reshape(batchedMul(Wv28, Xna28), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc28 = reshapeBatch(Kr28, {hd, T, HkvB, 1});
+Krt28 = tile(Krc28, {1, 1, 1, HkvMul});
+Krp28 = permute(Krt28, {0, 1, 3, 2});
+Kt28 = reshapeBatch(Krp28, {hd, T, 1, HB});
+Vrc28 = reshapeBatch(V28, {hd, T, HkvB, 1});
+Vrt28 = tile(Vrc28, {1, 1, 1, HkvMul});
+Vrp28 = permute(Vrt28, {0, 1, 3, 2});
+Vt28 = reshapeBatch(Vrp28, {hd, T, 1, HB});
+A28 = attention(Qr28, Kt28, Vt28, hd, 1);
+Wo28 = MatrixWithName("gemm_199.weight", D, Dq, 1, 1);
+O28 = batchedMul(Wo28, reshape(permute(A28, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R28 = X28 + O28;
+Wrf28 = MatrixWithName("rmsn_396.weight", D, 1);
+Rfn28 = rmsNorm(R28, Wrf28, 1e-5);
+Wg28 = MatrixWithName("gemm_200.weight", I, D, 1, 1);
+Wu28 = MatrixWithName("gemm_201.weight", I, D, 1, 1);
+Wd28 = MatrixWithName("gemm_202.weight", D, I, 1, 1);
+X29 = R28 + batchedMul(Wd28, elementMul(silu(batchedMul(Wg28, Rfn28)), batchedMul(Wu28, Rfn28)));
+Wra29 = MatrixWithName("rmsn_397.weight", D, 1);
+Xna29 = rmsNorm(X29, Wra29, 1e-5);
+Wq29 = MatrixWithName("gemm_203.weight", Dq, D, 1, 1);
+Qh29 = permute(reshape(batchedMul(Wq29, Xna29), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn29 = MatrixWithName("rmsn_398.weight", hd, 1);
+Qn29 = rmsNorm(Qh29, Wqn29, 1e-5);
+Qr29 = rope(Qn29, cos_te, sin_te);
+Wk29 = MatrixWithName("gemm_204.weight", Dkv, D, 1, 1);
+Kh29 = permute(reshape(batchedMul(Wk29, Xna29), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn29 = MatrixWithName("rmsn_399.weight", hd, 1);
+Kn29 = rmsNorm(Kh29, Wkn29, 1e-5);
+Kr29 = rope(Kn29, cos_te, sin_te);
+Wv29 = MatrixWithName("gemm_205.weight", Dkv, D, 1, 1);
+V29 = permute(reshape(batchedMul(Wv29, Xna29), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc29 = reshapeBatch(Kr29, {hd, T, HkvB, 1});
+Krt29 = tile(Krc29, {1, 1, 1, HkvMul});
+Krp29 = permute(Krt29, {0, 1, 3, 2});
+Kt29 = reshapeBatch(Krp29, {hd, T, 1, HB});
+Vrc29 = reshapeBatch(V29, {hd, T, HkvB, 1});
+Vrt29 = tile(Vrc29, {1, 1, 1, HkvMul});
+Vrp29 = permute(Vrt29, {0, 1, 3, 2});
+Vt29 = reshapeBatch(Vrp29, {hd, T, 1, HB});
+A29 = attention(Qr29, Kt29, Vt29, hd, 1);
+Wo29 = MatrixWithName("gemm_206.weight", D, Dq, 1, 1);
+O29 = batchedMul(Wo29, reshape(permute(A29, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R29 = X29 + O29;
+Wrf29 = MatrixWithName("rmsn_400.weight", D, 1);
+Rfn29 = rmsNorm(R29, Wrf29, 1e-5);
+Wg29 = MatrixWithName("gemm_207.weight", I, D, 1, 1);
+Wu29 = MatrixWithName("gemm_208.weight", I, D, 1, 1);
+Wd29 = MatrixWithName("gemm_209.weight", D, I, 1, 1);
+X30 = R29 + batchedMul(Wd29, elementMul(silu(batchedMul(Wg29, Rfn29)), batchedMul(Wu29, Rfn29)));
+Wra30 = MatrixWithName("rmsn_401.weight", D, 1);
+Xna30 = rmsNorm(X30, Wra30, 1e-5);
+Wq30 = MatrixWithName("gemm_210.weight", Dq, D, 1, 1);
+Qh30 = permute(reshape(batchedMul(Wq30, Xna30), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn30 = MatrixWithName("rmsn_402.weight", hd, 1);
+Qn30 = rmsNorm(Qh30, Wqn30, 1e-5);
+Qr30 = rope(Qn30, cos_te, sin_te);
+Wk30 = MatrixWithName("gemm_211.weight", Dkv, D, 1, 1);
+Kh30 = permute(reshape(batchedMul(Wk30, Xna30), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn30 = MatrixWithName("rmsn_403.weight", hd, 1);
+Kn30 = rmsNorm(Kh30, Wkn30, 1e-5);
+Kr30 = rope(Kn30, cos_te, sin_te);
+Wv30 = MatrixWithName("gemm_212.weight", Dkv, D, 1, 1);
+V30 = permute(reshape(batchedMul(Wv30, Xna30), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc30 = reshapeBatch(Kr30, {hd, T, HkvB, 1});
+Krt30 = tile(Krc30, {1, 1, 1, HkvMul});
+Krp30 = permute(Krt30, {0, 1, 3, 2});
+Kt30 = reshapeBatch(Krp30, {hd, T, 1, HB});
+Vrc30 = reshapeBatch(V30, {hd, T, HkvB, 1});
+Vrt30 = tile(Vrc30, {1, 1, 1, HkvMul});
+Vrp30 = permute(Vrt30, {0, 1, 3, 2});
+Vt30 = reshapeBatch(Vrp30, {hd, T, 1, HB});
+A30 = attention(Qr30, Kt30, Vt30, hd, 1);
+Wo30 = MatrixWithName("gemm_213.weight", D, Dq, 1, 1);
+O30 = batchedMul(Wo30, reshape(permute(A30, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R30 = X30 + O30;
+Wrf30 = MatrixWithName("rmsn_404.weight", D, 1);
+Rfn30 = rmsNorm(R30, Wrf30, 1e-5);
+Wg30 = MatrixWithName("gemm_214.weight", I, D, 1, 1);
+Wu30 = MatrixWithName("gemm_215.weight", I, D, 1, 1);
+Wd30 = MatrixWithName("gemm_216.weight", D, I, 1, 1);
+X31 = R30 + batchedMul(Wd30, elementMul(silu(batchedMul(Wg30, Rfn30)), batchedMul(Wu30, Rfn30)));
+Wra31 = MatrixWithName("rmsn_405.weight", D, 1);
+Xna31 = rmsNorm(X31, Wra31, 1e-5);
+Wq31 = MatrixWithName("gemm_217.weight", Dq, D, 1, 1);
+Qh31 = permute(reshape(batchedMul(Wq31, Xna31), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn31 = MatrixWithName("rmsn_406.weight", hd, 1);
+Qn31 = rmsNorm(Qh31, Wqn31, 1e-5);
+Qr31 = rope(Qn31, cos_te, sin_te);
+Wk31 = MatrixWithName("gemm_218.weight", Dkv, D, 1, 1);
+Kh31 = permute(reshape(batchedMul(Wk31, Xna31), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn31 = MatrixWithName("rmsn_407.weight", hd, 1);
+Kn31 = rmsNorm(Kh31, Wkn31, 1e-5);
+Kr31 = rope(Kn31, cos_te, sin_te);
+Wv31 = MatrixWithName("gemm_219.weight", Dkv, D, 1, 1);
+V31 = permute(reshape(batchedMul(Wv31, Xna31), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc31 = reshapeBatch(Kr31, {hd, T, HkvB, 1});
+Krt31 = tile(Krc31, {1, 1, 1, HkvMul});
+Krp31 = permute(Krt31, {0, 1, 3, 2});
+Kt31 = reshapeBatch(Krp31, {hd, T, 1, HB});
+Vrc31 = reshapeBatch(V31, {hd, T, HkvB, 1});
+Vrt31 = tile(Vrc31, {1, 1, 1, HkvMul});
+Vrp31 = permute(Vrt31, {0, 1, 3, 2});
+Vt31 = reshapeBatch(Vrp31, {hd, T, 1, HB});
+A31 = attention(Qr31, Kt31, Vt31, hd, 1);
+Wo31 = MatrixWithName("gemm_220.weight", D, Dq, 1, 1);
+O31 = batchedMul(Wo31, reshape(permute(A31, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R31 = X31 + O31;
+Wrf31 = MatrixWithName("rmsn_408.weight", D, 1);
+Rfn31 = rmsNorm(R31, Wrf31, 1e-5);
+Wg31 = MatrixWithName("gemm_221.weight", I, D, 1, 1);
+Wu31 = MatrixWithName("gemm_222.weight", I, D, 1, 1);
+Wd31 = MatrixWithName("gemm_223.weight", D, I, 1, 1);
+X32 = R31 + batchedMul(Wd31, elementMul(silu(batchedMul(Wg31, Rfn31)), batchedMul(Wu31, Rfn31)));
+Wra32 = MatrixWithName("rmsn_409.weight", D, 1);
+Xna32 = rmsNorm(X32, Wra32, 1e-5);
+Wq32 = MatrixWithName("gemm_224.weight", Dq, D, 1, 1);
+Qh32 = permute(reshape(batchedMul(Wq32, Xna32), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn32 = MatrixWithName("rmsn_410.weight", hd, 1);
+Qn32 = rmsNorm(Qh32, Wqn32, 1e-5);
+Qr32 = rope(Qn32, cos_te, sin_te);
+Wk32 = MatrixWithName("gemm_225.weight", Dkv, D, 1, 1);
+Kh32 = permute(reshape(batchedMul(Wk32, Xna32), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn32 = MatrixWithName("rmsn_411.weight", hd, 1);
+Kn32 = rmsNorm(Kh32, Wkn32, 1e-5);
+Kr32 = rope(Kn32, cos_te, sin_te);
+Wv32 = MatrixWithName("gemm_226.weight", Dkv, D, 1, 1);
+V32 = permute(reshape(batchedMul(Wv32, Xna32), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc32 = reshapeBatch(Kr32, {hd, T, HkvB, 1});
+Krt32 = tile(Krc32, {1, 1, 1, HkvMul});
+Krp32 = permute(Krt32, {0, 1, 3, 2});
+Kt32 = reshapeBatch(Krp32, {hd, T, 1, HB});
+Vrc32 = reshapeBatch(V32, {hd, T, HkvB, 1});
+Vrt32 = tile(Vrc32, {1, 1, 1, HkvMul});
+Vrp32 = permute(Vrt32, {0, 1, 3, 2});
+Vt32 = reshapeBatch(Vrp32, {hd, T, 1, HB});
+A32 = attention(Qr32, Kt32, Vt32, hd, 1);
+Wo32 = MatrixWithName("gemm_227.weight", D, Dq, 1, 1);
+O32 = batchedMul(Wo32, reshape(permute(A32, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R32 = X32 + O32;
+Wrf32 = MatrixWithName("rmsn_412.weight", D, 1);
+Rfn32 = rmsNorm(R32, Wrf32, 1e-5);
+Wg32 = MatrixWithName("gemm_228.weight", I, D, 1, 1);
+Wu32 = MatrixWithName("gemm_229.weight", I, D, 1, 1);
+Wd32 = MatrixWithName("gemm_230.weight", D, I, 1, 1);
+X33 = R32 + batchedMul(Wd32, elementMul(silu(batchedMul(Wg32, Rfn32)), batchedMul(Wu32, Rfn32)));
+Wra33 = MatrixWithName("rmsn_413.weight", D, 1);
+Xna33 = rmsNorm(X33, Wra33, 1e-5);
+Wq33 = MatrixWithName("gemm_231.weight", Dq, D, 1, 1);
+Qh33 = permute(reshape(batchedMul(Wq33, Xna33), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn33 = MatrixWithName("rmsn_414.weight", hd, 1);
+Qn33 = rmsNorm(Qh33, Wqn33, 1e-5);
+Qr33 = rope(Qn33, cos_te, sin_te);
+Wk33 = MatrixWithName("gemm_232.weight", Dkv, D, 1, 1);
+Kh33 = permute(reshape(batchedMul(Wk33, Xna33), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn33 = MatrixWithName("rmsn_415.weight", hd, 1);
+Kn33 = rmsNorm(Kh33, Wkn33, 1e-5);
+Kr33 = rope(Kn33, cos_te, sin_te);
+Wv33 = MatrixWithName("gemm_233.weight", Dkv, D, 1, 1);
+V33 = permute(reshape(batchedMul(Wv33, Xna33), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc33 = reshapeBatch(Kr33, {hd, T, HkvB, 1});
+Krt33 = tile(Krc33, {1, 1, 1, HkvMul});
+Krp33 = permute(Krt33, {0, 1, 3, 2});
+Kt33 = reshapeBatch(Krp33, {hd, T, 1, HB});
+Vrc33 = reshapeBatch(V33, {hd, T, HkvB, 1});
+Vrt33 = tile(Vrc33, {1, 1, 1, HkvMul});
+Vrp33 = permute(Vrt33, {0, 1, 3, 2});
+Vt33 = reshapeBatch(Vrp33, {hd, T, 1, HB});
+A33 = attention(Qr33, Kt33, Vt33, hd, 1);
+Wo33 = MatrixWithName("gemm_234.weight", D, Dq, 1, 1);
+O33 = batchedMul(Wo33, reshape(permute(A33, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R33 = X33 + O33;
+Wrf33 = MatrixWithName("rmsn_416.weight", D, 1);
+Rfn33 = rmsNorm(R33, Wrf33, 1e-5);
+Wg33 = MatrixWithName("gemm_235.weight", I, D, 1, 1);
+Wu33 = MatrixWithName("gemm_236.weight", I, D, 1, 1);
+Wd33 = MatrixWithName("gemm_237.weight", D, I, 1, 1);
+X34 = R33 + batchedMul(Wd33, elementMul(silu(batchedMul(Wg33, Rfn33)), batchedMul(Wu33, Rfn33)));
+Wra34 = MatrixWithName("rmsn_417.weight", D, 1);
+Xna34 = rmsNorm(X34, Wra34, 1e-5);
+Wq34 = MatrixWithName("gemm_238.weight", Dq, D, 1, 1);
+Qh34 = permute(reshape(batchedMul(Wq34, Xna34), {hd, HB, T, 1}), {0, 2, 3, 1});
+Wqn34 = MatrixWithName("rmsn_418.weight", hd, 1);
+Qn34 = rmsNorm(Qh34, Wqn34, 1e-5);
+Qr34 = rope(Qn34, cos_te, sin_te);
+Wk34 = MatrixWithName("gemm_239.weight", Dkv, D, 1, 1);
+Kh34 = permute(reshape(batchedMul(Wk34, Xna34), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Wkn34 = MatrixWithName("rmsn_419.weight", hd, 1);
+Kn34 = rmsNorm(Kh34, Wkn34, 1e-5);
+Kr34 = rope(Kn34, cos_te, sin_te);
+Wv34 = MatrixWithName("gemm_240.weight", Dkv, D, 1, 1);
+V34 = permute(reshape(batchedMul(Wv34, Xna34), {hd, HkvB, T, 1}), {0, 2, 3, 1});
+Krc34 = reshapeBatch(Kr34, {hd, T, HkvB, 1});
+Krt34 = tile(Krc34, {1, 1, 1, HkvMul});
+Krp34 = permute(Krt34, {0, 1, 3, 2});
+Kt34 = reshapeBatch(Krp34, {hd, T, 1, HB});
+Vrc34 = reshapeBatch(V34, {hd, T, HkvB, 1});
+Vrt34 = tile(Vrc34, {1, 1, 1, HkvMul});
+Vrp34 = permute(Vrt34, {0, 1, 3, 2});
+Vt34 = reshapeBatch(Vrp34, {hd, T, 1, HB});
+A34 = attention(Qr34, Kt34, Vt34, hd, 1);
+Wo34 = MatrixWithName("gemm_241.weight", D, Dq, 1, 1);
+O34 = batchedMul(Wo34, reshape(permute(A34, {0, 3, 1, 2}), {Dq, T, 1, 1}));
+R34 = X34 + O34;
+Wrf34 = MatrixWithName("rmsn_420.weight", D, 1);
+Rfn34 = rmsNorm(R34, Wrf34, 1e-5);
+Wg34 = MatrixWithName("gemm_242.weight", I, D, 1, 1);
+Wu34 = MatrixWithName("gemm_243.weight", I, D, 1, 1);
+Wd34 = MatrixWithName("gemm_244.weight", D, I, 1, 1);
+X35 = R34 + batchedMul(Wd34, elementMul(silu(batchedMul(Wg34, Rfn34)), batchedMul(Wu34, Rfn34)));
+
+setXY(ids, X35);
+'

net_texenc.ini.bak

@@ -0,0 +1,27 @@
+[train]
+load_net    = 1
+load_file=z_image_turbo_text_encoder.cccc.bin
+train_epochs = 0
+Batch       = 1
+gpu         = 1
+data_type = half
+output_log  = 0
+need_free_mem = 0
+
+[net]
+structure='
+B=1;
+X  = Matrix(768, T, 1, B);
+Ws = MatrixWithName("rmsn_1.weight", 768, 1);
+Xn = rmsNorm(X, Ws, 1e-5);
+W  = MatrixWithName("gemm_0.weight", 768, 768, 1, 1);
+b  = MatrixWithName("gemm_0.bias", 768, 1, 1, 1);
+Y  = batchedMul(W, Xn) + b;
+setXA(X, Y);
+'
+
+
+
+
+
+

net_unified.ini

@@ -0,0 +1,1227 @@
+[train]
+load_net    = 1
+load_file=z_image_turbo_transformer_unified.cccc.bin
+train_epochs = 0
+Batch       = 1
+gpu         = 1
+data_type = float
+output_log  = 0
+need_free_mem = 0
+
+[net]
+structure='
+B=1; D=3840; DD=3968; I=10240; hd=128; HB=30;
+Xin = Matrix(DD, SEQ+1, 1, B);
+Xin_pm = permute(Xin, {1, 0, 2, 3});
+t_slot = sliceW(Xin_pm, SEQ, 1);
+t_full = permute(t_slot, {1, 0, 2, 3});
+t_emb = sliceW(t_full, 0, 256);
+seq_pm = sliceW(Xin_pm, 0, SEQ);
+X0_wide = permute(seq_pm, {1, 0, 2, 3});
+X0 = sliceW(X0_wide, 0, D);
+cos1 = sliceW(X0_wide, D, 64);
+sin1 = sliceW(X0_wide, D+64, 64);
+Wmod0 = MatrixWithName("gemm_0.weight", 15360, 256, 1, 1);
+bmod0 = MatrixWithName("gemm_0.bias", 15360, 1, 1, 1);
+mod0 = batchedMul(Wmod0, t_emb) + bmod0;
+sa0 = sliceW(mod0, 0, D);
+ga0 = tanh(sliceW(mod0, D, D));
+sf0 = sliceW(mod0, 2*D, D);
+gf0 = tanh(sliceW(mod0, 3*D, D));
+Wrn330 = MatrixWithName("rmsn_330.weight", D, 1);
+Xn0 = rmsNorm(X0, Wrn330, 1e-5);
+Xm0 = Xn0 + elementMul(Xn0, sa0);
+Wq0 = MatrixWithName("gemm_1.weight", D, D, 1, 1);
+Wk0 = MatrixWithName("gemm_2.weight", D, D, 1, 1);
+Wv0 = MatrixWithName("gemm_3.weight", D, D, 1, 1);
+Wrn331 = MatrixWithName("rmsn_331.weight", hd, 1);
+Wrn332 = MatrixWithName("rmsn_332.weight", hd, 1);
+Wo0 = MatrixWithName("gemm_4.weight", D, D, 1, 1);
+Wrn333 = MatrixWithName("rmsn_333.weight", D, 1);
+Qr0 = permute(reshape(batchedMul(Wq0, Xm0), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr0 = permute(reshape(batchedMul(Wk0, Xm0), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr0 = permute(reshape(batchedMul(Wv0, Xm0), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn0 = rmsNorm(Qr0, Wrn331, 1e-5);
+Kn0 = rmsNorm(Kr0, Wrn332, 1e-5);
+At0 = attention(rope2(Qn0, cos1, sin1), rope2(Kn0, cos1, sin1), Vr0, hd, 0);
+Ao0 = batchedMul(Wo0, reshape(permute(At0, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An0 = rmsNorm(Ao0, Wrn333, 1e-5);
+Ag0 = elementMul(An0, ga0);
+Rx0 = X0 + Ag0;
+Wrn334 = MatrixWithName("rmsn_334.weight", D, 1);
+Wg0 = MatrixWithName("gemm_5.weight", I, D, 1, 1);
+Wu0 = MatrixWithName("gemm_6.weight", I, D, 1, 1);
+Wd0 = MatrixWithName("gemm_7.weight", D, I, 1, 1);
+Wrn335 = MatrixWithName("rmsn_335.weight", D, 1);
+Rf0 = rmsNorm(Rx0, Wrn334, 1e-5);
+Rfm0 = Rf0 + elementMul(Rf0, sf0);
+Fg0 = batchedMul(Wg0, Rfm0);
+Fu0 = batchedMul(Wu0, Rfm0);
+Ff0 = batchedMul(Wd0, elementMul(silu(Fg0), Fu0));
+Fn0 = rmsNorm(Ff0, Wrn335, 1e-5);
+Fgg0 = elementMul(Fn0, gf0);
+X1 = Rx0 + Fgg0;
+Wmod1 = MatrixWithName("gemm_8.weight", 15360, 256, 1, 1);
+bmod1 = MatrixWithName("gemm_8.bias", 15360, 1, 1, 1);
+mod1 = batchedMul(Wmod1, t_emb) + bmod1;
+sa1 = sliceW(mod1, 0, D);
+ga1 = tanh(sliceW(mod1, D, D));
+sf1 = sliceW(mod1, 2*D, D);
+gf1 = tanh(sliceW(mod1, 3*D, D));
+Wrn336 = MatrixWithName("rmsn_336.weight", D, 1);
+Xn1 = rmsNorm(X1, Wrn336, 1e-5);
+Xm1 = Xn1 + elementMul(Xn1, sa1);
+Wq1 = MatrixWithName("gemm_9.weight", D, D, 1, 1);
+Wk1 = MatrixWithName("gemm_10.weight", D, D, 1, 1);
+Wv1 = MatrixWithName("gemm_11.weight", D, D, 1, 1);
+Wrn337 = MatrixWithName("rmsn_337.weight", hd, 1);
+Wrn338 = MatrixWithName("rmsn_338.weight", hd, 1);
+Wo1 = MatrixWithName("gemm_12.weight", D, D, 1, 1);
+Wrn339 = MatrixWithName("rmsn_339.weight", D, 1);
+Qr1 = permute(reshape(batchedMul(Wq1, Xm1), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr1 = permute(reshape(batchedMul(Wk1, Xm1), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr1 = permute(reshape(batchedMul(Wv1, Xm1), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn1 = rmsNorm(Qr1, Wrn337, 1e-5);
+Kn1 = rmsNorm(Kr1, Wrn338, 1e-5);
+At1 = attention(rope2(Qn1, cos1, sin1), rope2(Kn1, cos1, sin1), Vr1, hd, 0);
+Ao1 = batchedMul(Wo1, reshape(permute(At1, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An1 = rmsNorm(Ao1, Wrn339, 1e-5);
+Ag1 = elementMul(An1, ga1);
+Rx1 = X1 + Ag1;
+Wrn340 = MatrixWithName("rmsn_340.weight", D, 1);
+Wg1 = MatrixWithName("gemm_13.weight", I, D, 1, 1);
+Wu1 = MatrixWithName("gemm_14.weight", I, D, 1, 1);
+Wd1 = MatrixWithName("gemm_15.weight", D, I, 1, 1);
+Wrn341 = MatrixWithName("rmsn_341.weight", D, 1);
+Rf1 = rmsNorm(Rx1, Wrn340, 1e-5);
+Rfm1 = Rf1 + elementMul(Rf1, sf1);
+Fg1 = batchedMul(Wg1, Rfm1);
+Fu1 = batchedMul(Wu1, Rfm1);
+Ff1 = batchedMul(Wd1, elementMul(silu(Fg1), Fu1));
+Fn1 = rmsNorm(Ff1, Wrn341, 1e-5);
+Fgg1 = elementMul(Fn1, gf1);
+X2 = Rx1 + Fgg1;
+Wmod2 = MatrixWithName("gemm_16.weight", 15360, 256, 1, 1);
+bmod2 = MatrixWithName("gemm_16.bias", 15360, 1, 1, 1);
+mod2 = batchedMul(Wmod2, t_emb) + bmod2;
+sa2 = sliceW(mod2, 0, D);
+ga2 = tanh(sliceW(mod2, D, D));
+sf2 = sliceW(mod2, 2*D, D);
+gf2 = tanh(sliceW(mod2, 3*D, D));
+Wrn342 = MatrixWithName("rmsn_342.weight", D, 1);
+Xn2 = rmsNorm(X2, Wrn342, 1e-5);
+Xm2 = Xn2 + elementMul(Xn2, sa2);
+Wq2 = MatrixWithName("gemm_17.weight", D, D, 1, 1);
+Wk2 = MatrixWithName("gemm_18.weight", D, D, 1, 1);
+Wv2 = MatrixWithName("gemm_19.weight", D, D, 1, 1);
+Wrn343 = MatrixWithName("rmsn_343.weight", hd, 1);
+Wrn344 = MatrixWithName("rmsn_344.weight", hd, 1);
+Wo2 = MatrixWithName("gemm_20.weight", D, D, 1, 1);
+Wrn345 = MatrixWithName("rmsn_345.weight", D, 1);
+Qr2 = permute(reshape(batchedMul(Wq2, Xm2), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr2 = permute(reshape(batchedMul(Wk2, Xm2), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr2 = permute(reshape(batchedMul(Wv2, Xm2), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn2 = rmsNorm(Qr2, Wrn343, 1e-5);
+Kn2 = rmsNorm(Kr2, Wrn344, 1e-5);
+At2 = attention(rope2(Qn2, cos1, sin1), rope2(Kn2, cos1, sin1), Vr2, hd, 0);
+Ao2 = batchedMul(Wo2, reshape(permute(At2, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An2 = rmsNorm(Ao2, Wrn345, 1e-5);
+Ag2 = elementMul(An2, ga2);
+Rx2 = X2 + Ag2;
+Wrn346 = MatrixWithName("rmsn_346.weight", D, 1);
+Wg2 = MatrixWithName("gemm_21.weight", I, D, 1, 1);
+Wu2 = MatrixWithName("gemm_22.weight", I, D, 1, 1);
+Wd2 = MatrixWithName("gemm_23.weight", D, I, 1, 1);
+Wrn347 = MatrixWithName("rmsn_347.weight", D, 1);
+Rf2 = rmsNorm(Rx2, Wrn346, 1e-5);
+Rfm2 = Rf2 + elementMul(Rf2, sf2);
+Fg2 = batchedMul(Wg2, Rfm2);
+Fu2 = batchedMul(Wu2, Rfm2);
+Ff2 = batchedMul(Wd2, elementMul(silu(Fg2), Fu2));
+Fn2 = rmsNorm(Ff2, Wrn347, 1e-5);
+Fgg2 = elementMul(Fn2, gf2);
+X3 = Rx2 + Fgg2;
+Wmod3 = MatrixWithName("gemm_24.weight", 15360, 256, 1, 1);
+bmod3 = MatrixWithName("gemm_24.bias", 15360, 1, 1, 1);
+mod3 = batchedMul(Wmod3, t_emb) + bmod3;
+sa3 = sliceW(mod3, 0, D);
+ga3 = tanh(sliceW(mod3, D, D));
+sf3 = sliceW(mod3, 2*D, D);
+gf3 = tanh(sliceW(mod3, 3*D, D));
+Wrn348 = MatrixWithName("rmsn_348.weight", D, 1);
+Xn3 = rmsNorm(X3, Wrn348, 1e-5);
+Xm3 = Xn3 + elementMul(Xn3, sa3);
+Wq3 = MatrixWithName("gemm_25.weight", D, D, 1, 1);
+Wk3 = MatrixWithName("gemm_26.weight", D, D, 1, 1);
+Wv3 = MatrixWithName("gemm_27.weight", D, D, 1, 1);
+Wrn349 = MatrixWithName("rmsn_349.weight", hd, 1);
+Wrn350 = MatrixWithName("rmsn_350.weight", hd, 1);
+Wo3 = MatrixWithName("gemm_28.weight", D, D, 1, 1);
+Wrn351 = MatrixWithName("rmsn_351.weight", D, 1);
+Qr3 = permute(reshape(batchedMul(Wq3, Xm3), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr3 = permute(reshape(batchedMul(Wk3, Xm3), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr3 = permute(reshape(batchedMul(Wv3, Xm3), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn3 = rmsNorm(Qr3, Wrn349, 1e-5);
+Kn3 = rmsNorm(Kr3, Wrn350, 1e-5);
+At3 = attention(rope2(Qn3, cos1, sin1), rope2(Kn3, cos1, sin1), Vr3, hd, 0);
+Ao3 = batchedMul(Wo3, reshape(permute(At3, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An3 = rmsNorm(Ao3, Wrn351, 1e-5);
+Ag3 = elementMul(An3, ga3);
+Rx3 = X3 + Ag3;
+Wrn352 = MatrixWithName("rmsn_352.weight", D, 1);
+Wg3 = MatrixWithName("gemm_29.weight", I, D, 1, 1);
+Wu3 = MatrixWithName("gemm_30.weight", I, D, 1, 1);
+Wd3 = MatrixWithName("gemm_31.weight", D, I, 1, 1);
+Wrn353 = MatrixWithName("rmsn_353.weight", D, 1);
+Rf3 = rmsNorm(Rx3, Wrn352, 1e-5);
+Rfm3 = Rf3 + elementMul(Rf3, sf3);
+Fg3 = batchedMul(Wg3, Rfm3);
+Fu3 = batchedMul(Wu3, Rfm3);
+Ff3 = batchedMul(Wd3, elementMul(silu(Fg3), Fu3));
+Fn3 = rmsNorm(Ff3, Wrn353, 1e-5);
+Fgg3 = elementMul(Fn3, gf3);
+X4 = Rx3 + Fgg3;
+Wmod4 = MatrixWithName("gemm_32.weight", 15360, 256, 1, 1);
+bmod4 = MatrixWithName("gemm_32.bias", 15360, 1, 1, 1);
+mod4 = batchedMul(Wmod4, t_emb) + bmod4;
+sa4 = sliceW(mod4, 0, D);
+ga4 = tanh(sliceW(mod4, D, D));
+sf4 = sliceW(mod4, 2*D, D);
+gf4 = tanh(sliceW(mod4, 3*D, D));
+Wrn354 = MatrixWithName("rmsn_354.weight", D, 1);
+Xn4 = rmsNorm(X4, Wrn354, 1e-5);
+Xm4 = Xn4 + elementMul(Xn4, sa4);
+Wq4 = MatrixWithName("gemm_33.weight", D, D, 1, 1);
+Wk4 = MatrixWithName("gemm_34.weight", D, D, 1, 1);
+Wv4 = MatrixWithName("gemm_35.weight", D, D, 1, 1);
+Wrn355 = MatrixWithName("rmsn_355.weight", hd, 1);
+Wrn356 = MatrixWithName("rmsn_356.weight", hd, 1);
+Wo4 = MatrixWithName("gemm_36.weight", D, D, 1, 1);
+Wrn357 = MatrixWithName("rmsn_357.weight", D, 1);
+Qr4 = permute(reshape(batchedMul(Wq4, Xm4), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr4 = permute(reshape(batchedMul(Wk4, Xm4), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr4 = permute(reshape(batchedMul(Wv4, Xm4), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn4 = rmsNorm(Qr4, Wrn355, 1e-5);
+Kn4 = rmsNorm(Kr4, Wrn356, 1e-5);
+At4 = attention(rope2(Qn4, cos1, sin1), rope2(Kn4, cos1, sin1), Vr4, hd, 0);
+Ao4 = batchedMul(Wo4, reshape(permute(At4, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An4 = rmsNorm(Ao4, Wrn357, 1e-5);
+Ag4 = elementMul(An4, ga4);
+Rx4 = X4 + Ag4;
+Wrn358 = MatrixWithName("rmsn_358.weight", D, 1);
+Wg4 = MatrixWithName("gemm_37.weight", I, D, 1, 1);
+Wu4 = MatrixWithName("gemm_38.weight", I, D, 1, 1);
+Wd4 = MatrixWithName("gemm_39.weight", D, I, 1, 1);
+Wrn359 = MatrixWithName("rmsn_359.weight", D, 1);
+Rf4 = rmsNorm(Rx4, Wrn358, 1e-5);
+Rfm4 = Rf4 + elementMul(Rf4, sf4);
+Fg4 = batchedMul(Wg4, Rfm4);
+Fu4 = batchedMul(Wu4, Rfm4);
+Ff4 = batchedMul(Wd4, elementMul(silu(Fg4), Fu4));
+Fn4 = rmsNorm(Ff4, Wrn359, 1e-5);
+Fgg4 = elementMul(Fn4, gf4);
+X5 = Rx4 + Fgg4;
+Wmod5 = MatrixWithName("gemm_40.weight", 15360, 256, 1, 1);
+bmod5 = MatrixWithName("gemm_40.bias", 15360, 1, 1, 1);
+mod5 = batchedMul(Wmod5, t_emb) + bmod5;
+sa5 = sliceW(mod5, 0, D);
+ga5 = tanh(sliceW(mod5, D, D));
+sf5 = sliceW(mod5, 2*D, D);
+gf5 = tanh(sliceW(mod5, 3*D, D));
+Wrn360 = MatrixWithName("rmsn_360.weight", D, 1);
+Xn5 = rmsNorm(X5, Wrn360, 1e-5);
+Xm5 = Xn5 + elementMul(Xn5, sa5);
+Wq5 = MatrixWithName("gemm_41.weight", D, D, 1, 1);
+Wk5 = MatrixWithName("gemm_42.weight", D, D, 1, 1);
+Wv5 = MatrixWithName("gemm_43.weight", D, D, 1, 1);
+Wrn361 = MatrixWithName("rmsn_361.weight", hd, 1);
+Wrn362 = MatrixWithName("rmsn_362.weight", hd, 1);
+Wo5 = MatrixWithName("gemm_44.weight", D, D, 1, 1);
+Wrn363 = MatrixWithName("rmsn_363.weight", D, 1);
+Qr5 = permute(reshape(batchedMul(Wq5, Xm5), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr5 = permute(reshape(batchedMul(Wk5, Xm5), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr5 = permute(reshape(batchedMul(Wv5, Xm5), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn5 = rmsNorm(Qr5, Wrn361, 1e-5);
+Kn5 = rmsNorm(Kr5, Wrn362, 1e-5);
+At5 = attention(rope2(Qn5, cos1, sin1), rope2(Kn5, cos1, sin1), Vr5, hd, 0);
+Ao5 = batchedMul(Wo5, reshape(permute(At5, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An5 = rmsNorm(Ao5, Wrn363, 1e-5);
+Ag5 = elementMul(An5, ga5);
+Rx5 = X5 + Ag5;
+Wrn364 = MatrixWithName("rmsn_364.weight", D, 1);
+Wg5 = MatrixWithName("gemm_45.weight", I, D, 1, 1);
+Wu5 = MatrixWithName("gemm_46.weight", I, D, 1, 1);
+Wd5 = MatrixWithName("gemm_47.weight", D, I, 1, 1);
+Wrn365 = MatrixWithName("rmsn_365.weight", D, 1);
+Rf5 = rmsNorm(Rx5, Wrn364, 1e-5);
+Rfm5 = Rf5 + elementMul(Rf5, sf5);
+Fg5 = batchedMul(Wg5, Rfm5);
+Fu5 = batchedMul(Wu5, Rfm5);
+Ff5 = batchedMul(Wd5, elementMul(silu(Fg5), Fu5));
+Fn5 = rmsNorm(Ff5, Wrn365, 1e-5);
+Fgg5 = elementMul(Fn5, gf5);
+X6 = Rx5 + Fgg5;
+Wmod6 = MatrixWithName("gemm_48.weight", 15360, 256, 1, 1);
+bmod6 = MatrixWithName("gemm_48.bias", 15360, 1, 1, 1);
+mod6 = batchedMul(Wmod6, t_emb) + bmod6;
+sa6 = sliceW(mod6, 0, D);
+ga6 = tanh(sliceW(mod6, D, D));
+sf6 = sliceW(mod6, 2*D, D);
+gf6 = tanh(sliceW(mod6, 3*D, D));
+Wrn366 = MatrixWithName("rmsn_366.weight", D, 1);
+Xn6 = rmsNorm(X6, Wrn366, 1e-5);
+Xm6 = Xn6 + elementMul(Xn6, sa6);
+Wq6 = MatrixWithName("gemm_49.weight", D, D, 1, 1);
+Wk6 = MatrixWithName("gemm_50.weight", D, D, 1, 1);
+Wv6 = MatrixWithName("gemm_51.weight", D, D, 1, 1);
+Wrn367 = MatrixWithName("rmsn_367.weight", hd, 1);
+Wrn368 = MatrixWithName("rmsn_368.weight", hd, 1);
+Wo6 = MatrixWithName("gemm_52.weight", D, D, 1, 1);
+Wrn369 = MatrixWithName("rmsn_369.weight", D, 1);
+Qr6 = permute(reshape(batchedMul(Wq6, Xm6), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr6 = permute(reshape(batchedMul(Wk6, Xm6), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr6 = permute(reshape(batchedMul(Wv6, Xm6), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn6 = rmsNorm(Qr6, Wrn367, 1e-5);
+Kn6 = rmsNorm(Kr6, Wrn368, 1e-5);
+At6 = attention(rope2(Qn6, cos1, sin1), rope2(Kn6, cos1, sin1), Vr6, hd, 0);
+Ao6 = batchedMul(Wo6, reshape(permute(At6, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An6 = rmsNorm(Ao6, Wrn369, 1e-5);
+Ag6 = elementMul(An6, ga6);
+Rx6 = X6 + Ag6;
+Wrn370 = MatrixWithName("rmsn_370.weight", D, 1);
+Wg6 = MatrixWithName("gemm_53.weight", I, D, 1, 1);
+Wu6 = MatrixWithName("gemm_54.weight", I, D, 1, 1);
+Wd6 = MatrixWithName("gemm_55.weight", D, I, 1, 1);
+Wrn371 = MatrixWithName("rmsn_371.weight", D, 1);
+Rf6 = rmsNorm(Rx6, Wrn370, 1e-5);
+Rfm6 = Rf6 + elementMul(Rf6, sf6);
+Fg6 = batchedMul(Wg6, Rfm6);
+Fu6 = batchedMul(Wu6, Rfm6);
+Ff6 = batchedMul(Wd6, elementMul(silu(Fg6), Fu6));
+Fn6 = rmsNorm(Ff6, Wrn371, 1e-5);
+Fgg6 = elementMul(Fn6, gf6);
+X7 = Rx6 + Fgg6;
+Wmod7 = MatrixWithName("gemm_56.weight", 15360, 256, 1, 1);
+bmod7 = MatrixWithName("gemm_56.bias", 15360, 1, 1, 1);
+mod7 = batchedMul(Wmod7, t_emb) + bmod7;
+sa7 = sliceW(mod7, 0, D);
+ga7 = tanh(sliceW(mod7, D, D));
+sf7 = sliceW(mod7, 2*D, D);
+gf7 = tanh(sliceW(mod7, 3*D, D));
+Wrn372 = MatrixWithName("rmsn_372.weight", D, 1);
+Xn7 = rmsNorm(X7, Wrn372, 1e-5);
+Xm7 = Xn7 + elementMul(Xn7, sa7);
+Wq7 = MatrixWithName("gemm_57.weight", D, D, 1, 1);
+Wk7 = MatrixWithName("gemm_58.weight", D, D, 1, 1);
+Wv7 = MatrixWithName("gemm_59.weight", D, D, 1, 1);
+Wrn373 = MatrixWithName("rmsn_373.weight", hd, 1);
+Wrn374 = MatrixWithName("rmsn_374.weight", hd, 1);
+Wo7 = MatrixWithName("gemm_60.weight", D, D, 1, 1);
+Wrn375 = MatrixWithName("rmsn_375.weight", D, 1);
+Qr7 = permute(reshape(batchedMul(Wq7, Xm7), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr7 = permute(reshape(batchedMul(Wk7, Xm7), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr7 = permute(reshape(batchedMul(Wv7, Xm7), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn7 = rmsNorm(Qr7, Wrn373, 1e-5);
+Kn7 = rmsNorm(Kr7, Wrn374, 1e-5);
+At7 = attention(rope2(Qn7, cos1, sin1), rope2(Kn7, cos1, sin1), Vr7, hd, 0);
+Ao7 = batchedMul(Wo7, reshape(permute(At7, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An7 = rmsNorm(Ao7, Wrn375, 1e-5);
+Ag7 = elementMul(An7, ga7);
+Rx7 = X7 + Ag7;
+Wrn376 = MatrixWithName("rmsn_376.weight", D, 1);
+Wg7 = MatrixWithName("gemm_61.weight", I, D, 1, 1);
+Wu7 = MatrixWithName("gemm_62.weight", I, D, 1, 1);
+Wd7 = MatrixWithName("gemm_63.weight", D, I, 1, 1);
+Wrn377 = MatrixWithName("rmsn_377.weight", D, 1);
+Rf7 = rmsNorm(Rx7, Wrn376, 1e-5);
+Rfm7 = Rf7 + elementMul(Rf7, sf7);
+Fg7 = batchedMul(Wg7, Rfm7);
+Fu7 = batchedMul(Wu7, Rfm7);
+Ff7 = batchedMul(Wd7, elementMul(silu(Fg7), Fu7));
+Fn7 = rmsNorm(Ff7, Wrn377, 1e-5);
+Fgg7 = elementMul(Fn7, gf7);
+X8 = Rx7 + Fgg7;
+Wmod8 = MatrixWithName("gemm_64.weight", 15360, 256, 1, 1);
+bmod8 = MatrixWithName("gemm_64.bias", 15360, 1, 1, 1);
+mod8 = batchedMul(Wmod8, t_emb) + bmod8;
+sa8 = sliceW(mod8, 0, D);
+ga8 = tanh(sliceW(mod8, D, D));
+sf8 = sliceW(mod8, 2*D, D);
+gf8 = tanh(sliceW(mod8, 3*D, D));
+Wrn378 = MatrixWithName("rmsn_378.weight", D, 1);
+Xn8 = rmsNorm(X8, Wrn378, 1e-5);
+Xm8 = Xn8 + elementMul(Xn8, sa8);
+Wq8 = MatrixWithName("gemm_65.weight", D, D, 1, 1);
+Wk8 = MatrixWithName("gemm_66.weight", D, D, 1, 1);
+Wv8 = MatrixWithName("gemm_67.weight", D, D, 1, 1);
+Wrn379 = MatrixWithName("rmsn_379.weight", hd, 1);
+Wrn380 = MatrixWithName("rmsn_380.weight", hd, 1);
+Wo8 = MatrixWithName("gemm_68.weight", D, D, 1, 1);
+Wrn381 = MatrixWithName("rmsn_381.weight", D, 1);
+Qr8 = permute(reshape(batchedMul(Wq8, Xm8), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr8 = permute(reshape(batchedMul(Wk8, Xm8), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr8 = permute(reshape(batchedMul(Wv8, Xm8), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn8 = rmsNorm(Qr8, Wrn379, 1e-5);
+Kn8 = rmsNorm(Kr8, Wrn380, 1e-5);
+At8 = attention(rope2(Qn8, cos1, sin1), rope2(Kn8, cos1, sin1), Vr8, hd, 0);
+Ao8 = batchedMul(Wo8, reshape(permute(At8, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An8 = rmsNorm(Ao8, Wrn381, 1e-5);
+Ag8 = elementMul(An8, ga8);
+Rx8 = X8 + Ag8;
+Wrn382 = MatrixWithName("rmsn_382.weight", D, 1);
+Wg8 = MatrixWithName("gemm_69.weight", I, D, 1, 1);
+Wu8 = MatrixWithName("gemm_70.weight", I, D, 1, 1);
+Wd8 = MatrixWithName("gemm_71.weight", D, I, 1, 1);
+Wrn383 = MatrixWithName("rmsn_383.weight", D, 1);
+Rf8 = rmsNorm(Rx8, Wrn382, 1e-5);
+Rfm8 = Rf8 + elementMul(Rf8, sf8);
+Fg8 = batchedMul(Wg8, Rfm8);
+Fu8 = batchedMul(Wu8, Rfm8);
+Ff8 = batchedMul(Wd8, elementMul(silu(Fg8), Fu8));
+Fn8 = rmsNorm(Ff8, Wrn383, 1e-5);
+Fgg8 = elementMul(Fn8, gf8);
+X9 = Rx8 + Fgg8;
+Wmod9 = MatrixWithName("gemm_72.weight", 15360, 256, 1, 1);
+bmod9 = MatrixWithName("gemm_72.bias", 15360, 1, 1, 1);
+mod9 = batchedMul(Wmod9, t_emb) + bmod9;
+sa9 = sliceW(mod9, 0, D);
+ga9 = tanh(sliceW(mod9, D, D));
+sf9 = sliceW(mod9, 2*D, D);
+gf9 = tanh(sliceW(mod9, 3*D, D));
+Wrn384 = MatrixWithName("rmsn_384.weight", D, 1);
+Xn9 = rmsNorm(X9, Wrn384, 1e-5);
+Xm9 = Xn9 + elementMul(Xn9, sa9);
+Wq9 = MatrixWithName("gemm_73.weight", D, D, 1, 1);
+Wk9 = MatrixWithName("gemm_74.weight", D, D, 1, 1);
+Wv9 = MatrixWithName("gemm_75.weight", D, D, 1, 1);
+Wrn385 = MatrixWithName("rmsn_385.weight", hd, 1);
+Wrn386 = MatrixWithName("rmsn_386.weight", hd, 1);
+Wo9 = MatrixWithName("gemm_76.weight", D, D, 1, 1);
+Wrn387 = MatrixWithName("rmsn_387.weight", D, 1);
+Qr9 = permute(reshape(batchedMul(Wq9, Xm9), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr9 = permute(reshape(batchedMul(Wk9, Xm9), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr9 = permute(reshape(batchedMul(Wv9, Xm9), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn9 = rmsNorm(Qr9, Wrn385, 1e-5);
+Kn9 = rmsNorm(Kr9, Wrn386, 1e-5);
+At9 = attention(rope2(Qn9, cos1, sin1), rope2(Kn9, cos1, sin1), Vr9, hd, 0);
+Ao9 = batchedMul(Wo9, reshape(permute(At9, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An9 = rmsNorm(Ao9, Wrn387, 1e-5);
+Ag9 = elementMul(An9, ga9);
+Rx9 = X9 + Ag9;
+Wrn388 = MatrixWithName("rmsn_388.weight", D, 1);
+Wg9 = MatrixWithName("gemm_77.weight", I, D, 1, 1);
+Wu9 = MatrixWithName("gemm_78.weight", I, D, 1, 1);
+Wd9 = MatrixWithName("gemm_79.weight", D, I, 1, 1);
+Wrn389 = MatrixWithName("rmsn_389.weight", D, 1);
+Rf9 = rmsNorm(Rx9, Wrn388, 1e-5);
+Rfm9 = Rf9 + elementMul(Rf9, sf9);
+Fg9 = batchedMul(Wg9, Rfm9);
+Fu9 = batchedMul(Wu9, Rfm9);
+Ff9 = batchedMul(Wd9, elementMul(silu(Fg9), Fu9));
+Fn9 = rmsNorm(Ff9, Wrn389, 1e-5);
+Fgg9 = elementMul(Fn9, gf9);
+X10 = Rx9 + Fgg9;
+Wmod10 = MatrixWithName("gemm_80.weight", 15360, 256, 1, 1);
+bmod10 = MatrixWithName("gemm_80.bias", 15360, 1, 1, 1);
+mod10 = batchedMul(Wmod10, t_emb) + bmod10;
+sa10 = sliceW(mod10, 0, D);
+ga10 = tanh(sliceW(mod10, D, D));
+sf10 = sliceW(mod10, 2*D, D);
+gf10 = tanh(sliceW(mod10, 3*D, D));
+Wrn390 = MatrixWithName("rmsn_390.weight", D, 1);
+Xn10 = rmsNorm(X10, Wrn390, 1e-5);
+Xm10 = Xn10 + elementMul(Xn10, sa10);
+Wq10 = MatrixWithName("gemm_81.weight", D, D, 1, 1);
+Wk10 = MatrixWithName("gemm_82.weight", D, D, 1, 1);
+Wv10 = MatrixWithName("gemm_83.weight", D, D, 1, 1);
+Wrn391 = MatrixWithName("rmsn_391.weight", hd, 1);
+Wrn392 = MatrixWithName("rmsn_392.weight", hd, 1);
+Wo10 = MatrixWithName("gemm_84.weight", D, D, 1, 1);
+Wrn393 = MatrixWithName("rmsn_393.weight", D, 1);
+Qr10 = permute(reshape(batchedMul(Wq10, Xm10), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr10 = permute(reshape(batchedMul(Wk10, Xm10), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr10 = permute(reshape(batchedMul(Wv10, Xm10), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn10 = rmsNorm(Qr10, Wrn391, 1e-5);
+Kn10 = rmsNorm(Kr10, Wrn392, 1e-5);
+At10 = attention(rope2(Qn10, cos1, sin1), rope2(Kn10, cos1, sin1), Vr10, hd, 0);
+Ao10 = batchedMul(Wo10, reshape(permute(At10, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An10 = rmsNorm(Ao10, Wrn393, 1e-5);
+Ag10 = elementMul(An10, ga10);
+Rx10 = X10 + Ag10;
+Wrn394 = MatrixWithName("rmsn_394.weight", D, 1);
+Wg10 = MatrixWithName("gemm_85.weight", I, D, 1, 1);
+Wu10 = MatrixWithName("gemm_86.weight", I, D, 1, 1);
+Wd10 = MatrixWithName("gemm_87.weight", D, I, 1, 1);
+Wrn395 = MatrixWithName("rmsn_395.weight", D, 1);
+Rf10 = rmsNorm(Rx10, Wrn394, 1e-5);
+Rfm10 = Rf10 + elementMul(Rf10, sf10);
+Fg10 = batchedMul(Wg10, Rfm10);
+Fu10 = batchedMul(Wu10, Rfm10);
+Ff10 = batchedMul(Wd10, elementMul(silu(Fg10), Fu10));
+Fn10 = rmsNorm(Ff10, Wrn395, 1e-5);
+Fgg10 = elementMul(Fn10, gf10);
+X11 = Rx10 + Fgg10;
+Wmod11 = MatrixWithName("gemm_88.weight", 15360, 256, 1, 1);
+bmod11 = MatrixWithName("gemm_88.bias", 15360, 1, 1, 1);
+mod11 = batchedMul(Wmod11, t_emb) + bmod11;
+sa11 = sliceW(mod11, 0, D);
+ga11 = tanh(sliceW(mod11, D, D));
+sf11 = sliceW(mod11, 2*D, D);
+gf11 = tanh(sliceW(mod11, 3*D, D));
+Wrn396 = MatrixWithName("rmsn_396.weight", D, 1);
+Xn11 = rmsNorm(X11, Wrn396, 1e-5);
+Xm11 = Xn11 + elementMul(Xn11, sa11);
+Wq11 = MatrixWithName("gemm_89.weight", D, D, 1, 1);
+Wk11 = MatrixWithName("gemm_90.weight", D, D, 1, 1);
+Wv11 = MatrixWithName("gemm_91.weight", D, D, 1, 1);
+Wrn397 = MatrixWithName("rmsn_397.weight", hd, 1);
+Wrn398 = MatrixWithName("rmsn_398.weight", hd, 1);
+Wo11 = MatrixWithName("gemm_92.weight", D, D, 1, 1);
+Wrn399 = MatrixWithName("rmsn_399.weight", D, 1);
+Qr11 = permute(reshape(batchedMul(Wq11, Xm11), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr11 = permute(reshape(batchedMul(Wk11, Xm11), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr11 = permute(reshape(batchedMul(Wv11, Xm11), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn11 = rmsNorm(Qr11, Wrn397, 1e-5);
+Kn11 = rmsNorm(Kr11, Wrn398, 1e-5);
+At11 = attention(rope2(Qn11, cos1, sin1), rope2(Kn11, cos1, sin1), Vr11, hd, 0);
+Ao11 = batchedMul(Wo11, reshape(permute(At11, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An11 = rmsNorm(Ao11, Wrn399, 1e-5);
+Ag11 = elementMul(An11, ga11);
+Rx11 = X11 + Ag11;
+Wrn400 = MatrixWithName("rmsn_400.weight", D, 1);
+Wg11 = MatrixWithName("gemm_93.weight", I, D, 1, 1);
+Wu11 = MatrixWithName("gemm_94.weight", I, D, 1, 1);
+Wd11 = MatrixWithName("gemm_95.weight", D, I, 1, 1);
+Wrn401 = MatrixWithName("rmsn_401.weight", D, 1);
+Rf11 = rmsNorm(Rx11, Wrn400, 1e-5);
+Rfm11 = Rf11 + elementMul(Rf11, sf11);
+Fg11 = batchedMul(Wg11, Rfm11);
+Fu11 = batchedMul(Wu11, Rfm11);
+Ff11 = batchedMul(Wd11, elementMul(silu(Fg11), Fu11));
+Fn11 = rmsNorm(Ff11, Wrn401, 1e-5);
+Fgg11 = elementMul(Fn11, gf11);
+X12 = Rx11 + Fgg11;
+Wmod12 = MatrixWithName("gemm_96.weight", 15360, 256, 1, 1);
+bmod12 = MatrixWithName("gemm_96.bias", 15360, 1, 1, 1);
+mod12 = batchedMul(Wmod12, t_emb) + bmod12;
+sa12 = sliceW(mod12, 0, D);
+ga12 = tanh(sliceW(mod12, D, D));
+sf12 = sliceW(mod12, 2*D, D);
+gf12 = tanh(sliceW(mod12, 3*D, D));
+Wrn402 = MatrixWithName("rmsn_402.weight", D, 1);
+Xn12 = rmsNorm(X12, Wrn402, 1e-5);
+Xm12 = Xn12 + elementMul(Xn12, sa12);
+Wq12 = MatrixWithName("gemm_97.weight", D, D, 1, 1);
+Wk12 = MatrixWithName("gemm_98.weight", D, D, 1, 1);
+Wv12 = MatrixWithName("gemm_99.weight", D, D, 1, 1);
+Wrn403 = MatrixWithName("rmsn_403.weight", hd, 1);
+Wrn404 = MatrixWithName("rmsn_404.weight", hd, 1);
+Wo12 = MatrixWithName("gemm_100.weight", D, D, 1, 1);
+Wrn405 = MatrixWithName("rmsn_405.weight", D, 1);
+Qr12 = permute(reshape(batchedMul(Wq12, Xm12), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr12 = permute(reshape(batchedMul(Wk12, Xm12), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr12 = permute(reshape(batchedMul(Wv12, Xm12), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn12 = rmsNorm(Qr12, Wrn403, 1e-5);
+Kn12 = rmsNorm(Kr12, Wrn404, 1e-5);
+At12 = attention(rope2(Qn12, cos1, sin1), rope2(Kn12, cos1, sin1), Vr12, hd, 0);
+Ao12 = batchedMul(Wo12, reshape(permute(At12, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An12 = rmsNorm(Ao12, Wrn405, 1e-5);
+Ag12 = elementMul(An12, ga12);
+Rx12 = X12 + Ag12;
+Wrn406 = MatrixWithName("rmsn_406.weight", D, 1);
+Wg12 = MatrixWithName("gemm_101.weight", I, D, 1, 1);
+Wu12 = MatrixWithName("gemm_102.weight", I, D, 1, 1);
+Wd12 = MatrixWithName("gemm_103.weight", D, I, 1, 1);
+Wrn407 = MatrixWithName("rmsn_407.weight", D, 1);
+Rf12 = rmsNorm(Rx12, Wrn406, 1e-5);
+Rfm12 = Rf12 + elementMul(Rf12, sf12);
+Fg12 = batchedMul(Wg12, Rfm12);
+Fu12 = batchedMul(Wu12, Rfm12);
+Ff12 = batchedMul(Wd12, elementMul(silu(Fg12), Fu12));
+Fn12 = rmsNorm(Ff12, Wrn407, 1e-5);
+Fgg12 = elementMul(Fn12, gf12);
+X13 = Rx12 + Fgg12;
+Wmod13 = MatrixWithName("gemm_104.weight", 15360, 256, 1, 1);
+bmod13 = MatrixWithName("gemm_104.bias", 15360, 1, 1, 1);
+mod13 = batchedMul(Wmod13, t_emb) + bmod13;
+sa13 = sliceW(mod13, 0, D);
+ga13 = tanh(sliceW(mod13, D, D));
+sf13 = sliceW(mod13, 2*D, D);
+gf13 = tanh(sliceW(mod13, 3*D, D));
+Wrn408 = MatrixWithName("rmsn_408.weight", D, 1);
+Xn13 = rmsNorm(X13, Wrn408, 1e-5);
+Xm13 = Xn13 + elementMul(Xn13, sa13);
+Wq13 = MatrixWithName("gemm_105.weight", D, D, 1, 1);
+Wk13 = MatrixWithName("gemm_106.weight", D, D, 1, 1);
+Wv13 = MatrixWithName("gemm_107.weight", D, D, 1, 1);
+Wrn409 = MatrixWithName("rmsn_409.weight", hd, 1);
+Wrn410 = MatrixWithName("rmsn_410.weight", hd, 1);
+Wo13 = MatrixWithName("gemm_108.weight", D, D, 1, 1);
+Wrn411 = MatrixWithName("rmsn_411.weight", D, 1);
+Qr13 = permute(reshape(batchedMul(Wq13, Xm13), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr13 = permute(reshape(batchedMul(Wk13, Xm13), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr13 = permute(reshape(batchedMul(Wv13, Xm13), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn13 = rmsNorm(Qr13, Wrn409, 1e-5);
+Kn13 = rmsNorm(Kr13, Wrn410, 1e-5);
+At13 = attention(rope2(Qn13, cos1, sin1), rope2(Kn13, cos1, sin1), Vr13, hd, 0);
+Ao13 = batchedMul(Wo13, reshape(permute(At13, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An13 = rmsNorm(Ao13, Wrn411, 1e-5);
+Ag13 = elementMul(An13, ga13);
+Rx13 = X13 + Ag13;
+Wrn412 = MatrixWithName("rmsn_412.weight", D, 1);
+Wg13 = MatrixWithName("gemm_109.weight", I, D, 1, 1);
+Wu13 = MatrixWithName("gemm_110.weight", I, D, 1, 1);
+Wd13 = MatrixWithName("gemm_111.weight", D, I, 1, 1);
+Wrn413 = MatrixWithName("rmsn_413.weight", D, 1);
+Rf13 = rmsNorm(Rx13, Wrn412, 1e-5);
+Rfm13 = Rf13 + elementMul(Rf13, sf13);
+Fg13 = batchedMul(Wg13, Rfm13);
+Fu13 = batchedMul(Wu13, Rfm13);
+Ff13 = batchedMul(Wd13, elementMul(silu(Fg13), Fu13));
+Fn13 = rmsNorm(Ff13, Wrn413, 1e-5);
+Fgg13 = elementMul(Fn13, gf13);
+X14 = Rx13 + Fgg13;
+Wmod14 = MatrixWithName("gemm_112.weight", 15360, 256, 1, 1);
+bmod14 = MatrixWithName("gemm_112.bias", 15360, 1, 1, 1);
+mod14 = batchedMul(Wmod14, t_emb) + bmod14;
+sa14 = sliceW(mod14, 0, D);
+ga14 = tanh(sliceW(mod14, D, D));
+sf14 = sliceW(mod14, 2*D, D);
+gf14 = tanh(sliceW(mod14, 3*D, D));
+Wrn414 = MatrixWithName("rmsn_414.weight", D, 1);
+Xn14 = rmsNorm(X14, Wrn414, 1e-5);
+Xm14 = Xn14 + elementMul(Xn14, sa14);
+Wq14 = MatrixWithName("gemm_113.weight", D, D, 1, 1);
+Wk14 = MatrixWithName("gemm_114.weight", D, D, 1, 1);
+Wv14 = MatrixWithName("gemm_115.weight", D, D, 1, 1);
+Wrn415 = MatrixWithName("rmsn_415.weight", hd, 1);
+Wrn416 = MatrixWithName("rmsn_416.weight", hd, 1);
+Wo14 = MatrixWithName("gemm_116.weight", D, D, 1, 1);
+Wrn417 = MatrixWithName("rmsn_417.weight", D, 1);
+Qr14 = permute(reshape(batchedMul(Wq14, Xm14), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr14 = permute(reshape(batchedMul(Wk14, Xm14), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr14 = permute(reshape(batchedMul(Wv14, Xm14), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn14 = rmsNorm(Qr14, Wrn415, 1e-5);
+Kn14 = rmsNorm(Kr14, Wrn416, 1e-5);
+At14 = attention(rope2(Qn14, cos1, sin1), rope2(Kn14, cos1, sin1), Vr14, hd, 0);
+Ao14 = batchedMul(Wo14, reshape(permute(At14, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An14 = rmsNorm(Ao14, Wrn417, 1e-5);
+Ag14 = elementMul(An14, ga14);
+Rx14 = X14 + Ag14;
+Wrn418 = MatrixWithName("rmsn_418.weight", D, 1);
+Wg14 = MatrixWithName("gemm_117.weight", I, D, 1, 1);
+Wu14 = MatrixWithName("gemm_118.weight", I, D, 1, 1);
+Wd14 = MatrixWithName("gemm_119.weight", D, I, 1, 1);
+Wrn419 = MatrixWithName("rmsn_419.weight", D, 1);
+Rf14 = rmsNorm(Rx14, Wrn418, 1e-5);
+Rfm14 = Rf14 + elementMul(Rf14, sf14);
+Fg14 = batchedMul(Wg14, Rfm14);
+Fu14 = batchedMul(Wu14, Rfm14);
+Ff14 = batchedMul(Wd14, elementMul(silu(Fg14), Fu14));
+Fn14 = rmsNorm(Ff14, Wrn419, 1e-5);
+Fgg14 = elementMul(Fn14, gf14);
+X15 = Rx14 + Fgg14;
+Wmod15 = MatrixWithName("gemm_120.weight", 15360, 256, 1, 1);
+bmod15 = MatrixWithName("gemm_120.bias", 15360, 1, 1, 1);
+mod15 = batchedMul(Wmod15, t_emb) + bmod15;
+sa15 = sliceW(mod15, 0, D);
+ga15 = tanh(sliceW(mod15, D, D));
+sf15 = sliceW(mod15, 2*D, D);
+gf15 = tanh(sliceW(mod15, 3*D, D));
+Wrn420 = MatrixWithName("rmsn_420.weight", D, 1);
+Xn15 = rmsNorm(X15, Wrn420, 1e-5);
+Xm15 = Xn15 + elementMul(Xn15, sa15);
+Wq15 = MatrixWithName("gemm_121.weight", D, D, 1, 1);
+Wk15 = MatrixWithName("gemm_122.weight", D, D, 1, 1);
+Wv15 = MatrixWithName("gemm_123.weight", D, D, 1, 1);
+Wrn421 = MatrixWithName("rmsn_421.weight", hd, 1);
+Wrn422 = MatrixWithName("rmsn_422.weight", hd, 1);
+Wo15 = MatrixWithName("gemm_124.weight", D, D, 1, 1);
+Wrn423 = MatrixWithName("rmsn_423.weight", D, 1);
+Qr15 = permute(reshape(batchedMul(Wq15, Xm15), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr15 = permute(reshape(batchedMul(Wk15, Xm15), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr15 = permute(reshape(batchedMul(Wv15, Xm15), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn15 = rmsNorm(Qr15, Wrn421, 1e-5);
+Kn15 = rmsNorm(Kr15, Wrn422, 1e-5);
+At15 = attention(rope2(Qn15, cos1, sin1), rope2(Kn15, cos1, sin1), Vr15, hd, 0);
+Ao15 = batchedMul(Wo15, reshape(permute(At15, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An15 = rmsNorm(Ao15, Wrn423, 1e-5);
+Ag15 = elementMul(An15, ga15);
+Rx15 = X15 + Ag15;
+Wrn424 = MatrixWithName("rmsn_424.weight", D, 1);
+Wg15 = MatrixWithName("gemm_125.weight", I, D, 1, 1);
+Wu15 = MatrixWithName("gemm_126.weight", I, D, 1, 1);
+Wd15 = MatrixWithName("gemm_127.weight", D, I, 1, 1);
+Wrn425 = MatrixWithName("rmsn_425.weight", D, 1);
+Rf15 = rmsNorm(Rx15, Wrn424, 1e-5);
+Rfm15 = Rf15 + elementMul(Rf15, sf15);
+Fg15 = batchedMul(Wg15, Rfm15);
+Fu15 = batchedMul(Wu15, Rfm15);
+Ff15 = batchedMul(Wd15, elementMul(silu(Fg15), Fu15));
+Fn15 = rmsNorm(Ff15, Wrn425, 1e-5);
+Fgg15 = elementMul(Fn15, gf15);
+X16 = Rx15 + Fgg15;
+Wmod16 = MatrixWithName("gemm_128.weight", 15360, 256, 1, 1);
+bmod16 = MatrixWithName("gemm_128.bias", 15360, 1, 1, 1);
+mod16 = batchedMul(Wmod16, t_emb) + bmod16;
+sa16 = sliceW(mod16, 0, D);
+ga16 = tanh(sliceW(mod16, D, D));
+sf16 = sliceW(mod16, 2*D, D);
+gf16 = tanh(sliceW(mod16, 3*D, D));
+Wrn426 = MatrixWithName("rmsn_426.weight", D, 1);
+Xn16 = rmsNorm(X16, Wrn426, 1e-5);
+Xm16 = Xn16 + elementMul(Xn16, sa16);
+Wq16 = MatrixWithName("gemm_129.weight", D, D, 1, 1);
+Wk16 = MatrixWithName("gemm_130.weight", D, D, 1, 1);
+Wv16 = MatrixWithName("gemm_131.weight", D, D, 1, 1);
+Wrn427 = MatrixWithName("rmsn_427.weight", hd, 1);
+Wrn428 = MatrixWithName("rmsn_428.weight", hd, 1);
+Wo16 = MatrixWithName("gemm_132.weight", D, D, 1, 1);
+Wrn429 = MatrixWithName("rmsn_429.weight", D, 1);
+Qr16 = permute(reshape(batchedMul(Wq16, Xm16), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr16 = permute(reshape(batchedMul(Wk16, Xm16), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr16 = permute(reshape(batchedMul(Wv16, Xm16), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn16 = rmsNorm(Qr16, Wrn427, 1e-5);
+Kn16 = rmsNorm(Kr16, Wrn428, 1e-5);
+At16 = attention(rope2(Qn16, cos1, sin1), rope2(Kn16, cos1, sin1), Vr16, hd, 0);
+Ao16 = batchedMul(Wo16, reshape(permute(At16, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An16 = rmsNorm(Ao16, Wrn429, 1e-5);
+Ag16 = elementMul(An16, ga16);
+Rx16 = X16 + Ag16;
+Wrn430 = MatrixWithName("rmsn_430.weight", D, 1);
+Wg16 = MatrixWithName("gemm_133.weight", I, D, 1, 1);
+Wu16 = MatrixWithName("gemm_134.weight", I, D, 1, 1);
+Wd16 = MatrixWithName("gemm_135.weight", D, I, 1, 1);
+Wrn431 = MatrixWithName("rmsn_431.weight", D, 1);
+Rf16 = rmsNorm(Rx16, Wrn430, 1e-5);
+Rfm16 = Rf16 + elementMul(Rf16, sf16);
+Fg16 = batchedMul(Wg16, Rfm16);
+Fu16 = batchedMul(Wu16, Rfm16);
+Ff16 = batchedMul(Wd16, elementMul(silu(Fg16), Fu16));
+Fn16 = rmsNorm(Ff16, Wrn431, 1e-5);
+Fgg16 = elementMul(Fn16, gf16);
+X17 = Rx16 + Fgg16;
+Wmod17 = MatrixWithName("gemm_136.weight", 15360, 256, 1, 1);
+bmod17 = MatrixWithName("gemm_136.bias", 15360, 1, 1, 1);
+mod17 = batchedMul(Wmod17, t_emb) + bmod17;
+sa17 = sliceW(mod17, 0, D);
+ga17 = tanh(sliceW(mod17, D, D));
+sf17 = sliceW(mod17, 2*D, D);
+gf17 = tanh(sliceW(mod17, 3*D, D));
+Wrn432 = MatrixWithName("rmsn_432.weight", D, 1);
+Xn17 = rmsNorm(X17, Wrn432, 1e-5);
+Xm17 = Xn17 + elementMul(Xn17, sa17);
+Wq17 = MatrixWithName("gemm_137.weight", D, D, 1, 1);
+Wk17 = MatrixWithName("gemm_138.weight", D, D, 1, 1);
+Wv17 = MatrixWithName("gemm_139.weight", D, D, 1, 1);
+Wrn433 = MatrixWithName("rmsn_433.weight", hd, 1);
+Wrn434 = MatrixWithName("rmsn_434.weight", hd, 1);
+Wo17 = MatrixWithName("gemm_140.weight", D, D, 1, 1);
+Wrn435 = MatrixWithName("rmsn_435.weight", D, 1);
+Qr17 = permute(reshape(batchedMul(Wq17, Xm17), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr17 = permute(reshape(batchedMul(Wk17, Xm17), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr17 = permute(reshape(batchedMul(Wv17, Xm17), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn17 = rmsNorm(Qr17, Wrn433, 1e-5);
+Kn17 = rmsNorm(Kr17, Wrn434, 1e-5);
+At17 = attention(rope2(Qn17, cos1, sin1), rope2(Kn17, cos1, sin1), Vr17, hd, 0);
+Ao17 = batchedMul(Wo17, reshape(permute(At17, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An17 = rmsNorm(Ao17, Wrn435, 1e-5);
+Ag17 = elementMul(An17, ga17);
+Rx17 = X17 + Ag17;
+Wrn436 = MatrixWithName("rmsn_436.weight", D, 1);
+Wg17 = MatrixWithName("gemm_141.weight", I, D, 1, 1);
+Wu17 = MatrixWithName("gemm_142.weight", I, D, 1, 1);
+Wd17 = MatrixWithName("gemm_143.weight", D, I, 1, 1);
+Wrn437 = MatrixWithName("rmsn_437.weight", D, 1);
+Rf17 = rmsNorm(Rx17, Wrn436, 1e-5);
+Rfm17 = Rf17 + elementMul(Rf17, sf17);
+Fg17 = batchedMul(Wg17, Rfm17);
+Fu17 = batchedMul(Wu17, Rfm17);
+Ff17 = batchedMul(Wd17, elementMul(silu(Fg17), Fu17));
+Fn17 = rmsNorm(Ff17, Wrn437, 1e-5);
+Fgg17 = elementMul(Fn17, gf17);
+X18 = Rx17 + Fgg17;
+Wmod18 = MatrixWithName("gemm_144.weight", 15360, 256, 1, 1);
+bmod18 = MatrixWithName("gemm_144.bias", 15360, 1, 1, 1);
+mod18 = batchedMul(Wmod18, t_emb) + bmod18;
+sa18 = sliceW(mod18, 0, D);
+ga18 = tanh(sliceW(mod18, D, D));
+sf18 = sliceW(mod18, 2*D, D);
+gf18 = tanh(sliceW(mod18, 3*D, D));
+Wrn438 = MatrixWithName("rmsn_438.weight", D, 1);
+Xn18 = rmsNorm(X18, Wrn438, 1e-5);
+Xm18 = Xn18 + elementMul(Xn18, sa18);
+Wq18 = MatrixWithName("gemm_145.weight", D, D, 1, 1);
+Wk18 = MatrixWithName("gemm_146.weight", D, D, 1, 1);
+Wv18 = MatrixWithName("gemm_147.weight", D, D, 1, 1);
+Wrn439 = MatrixWithName("rmsn_439.weight", hd, 1);
+Wrn440 = MatrixWithName("rmsn_440.weight", hd, 1);
+Wo18 = MatrixWithName("gemm_148.weight", D, D, 1, 1);
+Wrn441 = MatrixWithName("rmsn_441.weight", D, 1);
+Qr18 = permute(reshape(batchedMul(Wq18, Xm18), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr18 = permute(reshape(batchedMul(Wk18, Xm18), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr18 = permute(reshape(batchedMul(Wv18, Xm18), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn18 = rmsNorm(Qr18, Wrn439, 1e-5);
+Kn18 = rmsNorm(Kr18, Wrn440, 1e-5);
+At18 = attention(rope2(Qn18, cos1, sin1), rope2(Kn18, cos1, sin1), Vr18, hd, 0);
+Ao18 = batchedMul(Wo18, reshape(permute(At18, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An18 = rmsNorm(Ao18, Wrn441, 1e-5);
+Ag18 = elementMul(An18, ga18);
+Rx18 = X18 + Ag18;
+Wrn442 = MatrixWithName("rmsn_442.weight", D, 1);
+Wg18 = MatrixWithName("gemm_149.weight", I, D, 1, 1);
+Wu18 = MatrixWithName("gemm_150.weight", I, D, 1, 1);
+Wd18 = MatrixWithName("gemm_151.weight", D, I, 1, 1);
+Wrn443 = MatrixWithName("rmsn_443.weight", D, 1);
+Rf18 = rmsNorm(Rx18, Wrn442, 1e-5);
+Rfm18 = Rf18 + elementMul(Rf18, sf18);
+Fg18 = batchedMul(Wg18, Rfm18);
+Fu18 = batchedMul(Wu18, Rfm18);
+Ff18 = batchedMul(Wd18, elementMul(silu(Fg18), Fu18));
+Fn18 = rmsNorm(Ff18, Wrn443, 1e-5);
+Fgg18 = elementMul(Fn18, gf18);
+X19 = Rx18 + Fgg18;
+Wmod19 = MatrixWithName("gemm_152.weight", 15360, 256, 1, 1);
+bmod19 = MatrixWithName("gemm_152.bias", 15360, 1, 1, 1);
+mod19 = batchedMul(Wmod19, t_emb) + bmod19;
+sa19 = sliceW(mod19, 0, D);
+ga19 = tanh(sliceW(mod19, D, D));
+sf19 = sliceW(mod19, 2*D, D);
+gf19 = tanh(sliceW(mod19, 3*D, D));
+Wrn444 = MatrixWithName("rmsn_444.weight", D, 1);
+Xn19 = rmsNorm(X19, Wrn444, 1e-5);
+Xm19 = Xn19 + elementMul(Xn19, sa19);
+Wq19 = MatrixWithName("gemm_153.weight", D, D, 1, 1);
+Wk19 = MatrixWithName("gemm_154.weight", D, D, 1, 1);
+Wv19 = MatrixWithName("gemm_155.weight", D, D, 1, 1);
+Wrn445 = MatrixWithName("rmsn_445.weight", hd, 1);
+Wrn446 = MatrixWithName("rmsn_446.weight", hd, 1);
+Wo19 = MatrixWithName("gemm_156.weight", D, D, 1, 1);
+Wrn447 = MatrixWithName("rmsn_447.weight", D, 1);
+Qr19 = permute(reshape(batchedMul(Wq19, Xm19), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr19 = permute(reshape(batchedMul(Wk19, Xm19), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr19 = permute(reshape(batchedMul(Wv19, Xm19), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn19 = rmsNorm(Qr19, Wrn445, 1e-5);
+Kn19 = rmsNorm(Kr19, Wrn446, 1e-5);
+At19 = attention(rope2(Qn19, cos1, sin1), rope2(Kn19, cos1, sin1), Vr19, hd, 0);
+Ao19 = batchedMul(Wo19, reshape(permute(At19, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An19 = rmsNorm(Ao19, Wrn447, 1e-5);
+Ag19 = elementMul(An19, ga19);
+Rx19 = X19 + Ag19;
+Wrn448 = MatrixWithName("rmsn_448.weight", D, 1);
+Wg19 = MatrixWithName("gemm_157.weight", I, D, 1, 1);
+Wu19 = MatrixWithName("gemm_158.weight", I, D, 1, 1);
+Wd19 = MatrixWithName("gemm_159.weight", D, I, 1, 1);
+Wrn449 = MatrixWithName("rmsn_449.weight", D, 1);
+Rf19 = rmsNorm(Rx19, Wrn448, 1e-5);
+Rfm19 = Rf19 + elementMul(Rf19, sf19);
+Fg19 = batchedMul(Wg19, Rfm19);
+Fu19 = batchedMul(Wu19, Rfm19);
+Ff19 = batchedMul(Wd19, elementMul(silu(Fg19), Fu19));
+Fn19 = rmsNorm(Ff19, Wrn449, 1e-5);
+Fgg19 = elementMul(Fn19, gf19);
+X20 = Rx19 + Fgg19;
+Wmod20 = MatrixWithName("gemm_160.weight", 15360, 256, 1, 1);
+bmod20 = MatrixWithName("gemm_160.bias", 15360, 1, 1, 1);
+mod20 = batchedMul(Wmod20, t_emb) + bmod20;
+sa20 = sliceW(mod20, 0, D);
+ga20 = tanh(sliceW(mod20, D, D));
+sf20 = sliceW(mod20, 2*D, D);
+gf20 = tanh(sliceW(mod20, 3*D, D));
+Wrn450 = MatrixWithName("rmsn_450.weight", D, 1);
+Xn20 = rmsNorm(X20, Wrn450, 1e-5);
+Xm20 = Xn20 + elementMul(Xn20, sa20);
+Wq20 = MatrixWithName("gemm_161.weight", D, D, 1, 1);
+Wk20 = MatrixWithName("gemm_162.weight", D, D, 1, 1);
+Wv20 = MatrixWithName("gemm_163.weight", D, D, 1, 1);
+Wrn451 = MatrixWithName("rmsn_451.weight", hd, 1);
+Wrn452 = MatrixWithName("rmsn_452.weight", hd, 1);
+Wo20 = MatrixWithName("gemm_164.weight", D, D, 1, 1);
+Wrn453 = MatrixWithName("rmsn_453.weight", D, 1);
+Qr20 = permute(reshape(batchedMul(Wq20, Xm20), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr20 = permute(reshape(batchedMul(Wk20, Xm20), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr20 = permute(reshape(batchedMul(Wv20, Xm20), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn20 = rmsNorm(Qr20, Wrn451, 1e-5);
+Kn20 = rmsNorm(Kr20, Wrn452, 1e-5);
+At20 = attention(rope2(Qn20, cos1, sin1), rope2(Kn20, cos1, sin1), Vr20, hd, 0);
+Ao20 = batchedMul(Wo20, reshape(permute(At20, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An20 = rmsNorm(Ao20, Wrn453, 1e-5);
+Ag20 = elementMul(An20, ga20);
+Rx20 = X20 + Ag20;
+Wrn454 = MatrixWithName("rmsn_454.weight", D, 1);
+Wg20 = MatrixWithName("gemm_165.weight", I, D, 1, 1);
+Wu20 = MatrixWithName("gemm_166.weight", I, D, 1, 1);
+Wd20 = MatrixWithName("gemm_167.weight", D, I, 1, 1);
+Wrn455 = MatrixWithName("rmsn_455.weight", D, 1);
+Rf20 = rmsNorm(Rx20, Wrn454, 1e-5);
+Rfm20 = Rf20 + elementMul(Rf20, sf20);
+Fg20 = batchedMul(Wg20, Rfm20);
+Fu20 = batchedMul(Wu20, Rfm20);
+Ff20 = batchedMul(Wd20, elementMul(silu(Fg20), Fu20));
+Fn20 = rmsNorm(Ff20, Wrn455, 1e-5);
+Fgg20 = elementMul(Fn20, gf20);
+X21 = Rx20 + Fgg20;
+Wmod21 = MatrixWithName("gemm_168.weight", 15360, 256, 1, 1);
+bmod21 = MatrixWithName("gemm_168.bias", 15360, 1, 1, 1);
+mod21 = batchedMul(Wmod21, t_emb) + bmod21;
+sa21 = sliceW(mod21, 0, D);
+ga21 = tanh(sliceW(mod21, D, D));
+sf21 = sliceW(mod21, 2*D, D);
+gf21 = tanh(sliceW(mod21, 3*D, D));
+Wrn456 = MatrixWithName("rmsn_456.weight", D, 1);
+Xn21 = rmsNorm(X21, Wrn456, 1e-5);
+Xm21 = Xn21 + elementMul(Xn21, sa21);
+Wq21 = MatrixWithName("gemm_169.weight", D, D, 1, 1);
+Wk21 = MatrixWithName("gemm_170.weight", D, D, 1, 1);
+Wv21 = MatrixWithName("gemm_171.weight", D, D, 1, 1);
+Wrn457 = MatrixWithName("rmsn_457.weight", hd, 1);
+Wrn458 = MatrixWithName("rmsn_458.weight", hd, 1);
+Wo21 = MatrixWithName("gemm_172.weight", D, D, 1, 1);
+Wrn459 = MatrixWithName("rmsn_459.weight", D, 1);
+Qr21 = permute(reshape(batchedMul(Wq21, Xm21), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr21 = permute(reshape(batchedMul(Wk21, Xm21), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr21 = permute(reshape(batchedMul(Wv21, Xm21), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn21 = rmsNorm(Qr21, Wrn457, 1e-5);
+Kn21 = rmsNorm(Kr21, Wrn458, 1e-5);
+At21 = attention(rope2(Qn21, cos1, sin1), rope2(Kn21, cos1, sin1), Vr21, hd, 0);
+Ao21 = batchedMul(Wo21, reshape(permute(At21, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An21 = rmsNorm(Ao21, Wrn459, 1e-5);
+Ag21 = elementMul(An21, ga21);
+Rx21 = X21 + Ag21;
+Wrn460 = MatrixWithName("rmsn_460.weight", D, 1);
+Wg21 = MatrixWithName("gemm_173.weight", I, D, 1, 1);
+Wu21 = MatrixWithName("gemm_174.weight", I, D, 1, 1);
+Wd21 = MatrixWithName("gemm_175.weight", D, I, 1, 1);
+Wrn461 = MatrixWithName("rmsn_461.weight", D, 1);
+Rf21 = rmsNorm(Rx21, Wrn460, 1e-5);
+Rfm21 = Rf21 + elementMul(Rf21, sf21);
+Fg21 = batchedMul(Wg21, Rfm21);
+Fu21 = batchedMul(Wu21, Rfm21);
+Ff21 = batchedMul(Wd21, elementMul(silu(Fg21), Fu21));
+Fn21 = rmsNorm(Ff21, Wrn461, 1e-5);
+Fgg21 = elementMul(Fn21, gf21);
+X22 = Rx21 + Fgg21;
+Wmod22 = MatrixWithName("gemm_176.weight", 15360, 256, 1, 1);
+bmod22 = MatrixWithName("gemm_176.bias", 15360, 1, 1, 1);
+mod22 = batchedMul(Wmod22, t_emb) + bmod22;
+sa22 = sliceW(mod22, 0, D);
+ga22 = tanh(sliceW(mod22, D, D));
+sf22 = sliceW(mod22, 2*D, D);
+gf22 = tanh(sliceW(mod22, 3*D, D));
+Wrn462 = MatrixWithName("rmsn_462.weight", D, 1);
+Xn22 = rmsNorm(X22, Wrn462, 1e-5);
+Xm22 = Xn22 + elementMul(Xn22, sa22);
+Wq22 = MatrixWithName("gemm_177.weight", D, D, 1, 1);
+Wk22 = MatrixWithName("gemm_178.weight", D, D, 1, 1);
+Wv22 = MatrixWithName("gemm_179.weight", D, D, 1, 1);
+Wrn463 = MatrixWithName("rmsn_463.weight", hd, 1);
+Wrn464 = MatrixWithName("rmsn_464.weight", hd, 1);
+Wo22 = MatrixWithName("gemm_180.weight", D, D, 1, 1);
+Wrn465 = MatrixWithName("rmsn_465.weight", D, 1);
+Qr22 = permute(reshape(batchedMul(Wq22, Xm22), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr22 = permute(reshape(batchedMul(Wk22, Xm22), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr22 = permute(reshape(batchedMul(Wv22, Xm22), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn22 = rmsNorm(Qr22, Wrn463, 1e-5);
+Kn22 = rmsNorm(Kr22, Wrn464, 1e-5);
+At22 = attention(rope2(Qn22, cos1, sin1), rope2(Kn22, cos1, sin1), Vr22, hd, 0);
+Ao22 = batchedMul(Wo22, reshape(permute(At22, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An22 = rmsNorm(Ao22, Wrn465, 1e-5);
+Ag22 = elementMul(An22, ga22);
+Rx22 = X22 + Ag22;
+Wrn466 = MatrixWithName("rmsn_466.weight", D, 1);
+Wg22 = MatrixWithName("gemm_181.weight", I, D, 1, 1);
+Wu22 = MatrixWithName("gemm_182.weight", I, D, 1, 1);
+Wd22 = MatrixWithName("gemm_183.weight", D, I, 1, 1);
+Wrn467 = MatrixWithName("rmsn_467.weight", D, 1);
+Rf22 = rmsNorm(Rx22, Wrn466, 1e-5);
+Rfm22 = Rf22 + elementMul(Rf22, sf22);
+Fg22 = batchedMul(Wg22, Rfm22);
+Fu22 = batchedMul(Wu22, Rfm22);
+Ff22 = batchedMul(Wd22, elementMul(silu(Fg22), Fu22));
+Fn22 = rmsNorm(Ff22, Wrn467, 1e-5);
+Fgg22 = elementMul(Fn22, gf22);
+X23 = Rx22 + Fgg22;
+Wmod23 = MatrixWithName("gemm_184.weight", 15360, 256, 1, 1);
+bmod23 = MatrixWithName("gemm_184.bias", 15360, 1, 1, 1);
+mod23 = batchedMul(Wmod23, t_emb) + bmod23;
+sa23 = sliceW(mod23, 0, D);
+ga23 = tanh(sliceW(mod23, D, D));
+sf23 = sliceW(mod23, 2*D, D);
+gf23 = tanh(sliceW(mod23, 3*D, D));
+Wrn468 = MatrixWithName("rmsn_468.weight", D, 1);
+Xn23 = rmsNorm(X23, Wrn468, 1e-5);
+Xm23 = Xn23 + elementMul(Xn23, sa23);
+Wq23 = MatrixWithName("gemm_185.weight", D, D, 1, 1);
+Wk23 = MatrixWithName("gemm_186.weight", D, D, 1, 1);
+Wv23 = MatrixWithName("gemm_187.weight", D, D, 1, 1);
+Wrn469 = MatrixWithName("rmsn_469.weight", hd, 1);
+Wrn470 = MatrixWithName("rmsn_470.weight", hd, 1);
+Wo23 = MatrixWithName("gemm_188.weight", D, D, 1, 1);
+Wrn471 = MatrixWithName("rmsn_471.weight", D, 1);
+Qr23 = permute(reshape(batchedMul(Wq23, Xm23), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr23 = permute(reshape(batchedMul(Wk23, Xm23), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr23 = permute(reshape(batchedMul(Wv23, Xm23), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn23 = rmsNorm(Qr23, Wrn469, 1e-5);
+Kn23 = rmsNorm(Kr23, Wrn470, 1e-5);
+At23 = attention(rope2(Qn23, cos1, sin1), rope2(Kn23, cos1, sin1), Vr23, hd, 0);
+Ao23 = batchedMul(Wo23, reshape(permute(At23, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An23 = rmsNorm(Ao23, Wrn471, 1e-5);
+Ag23 = elementMul(An23, ga23);
+Rx23 = X23 + Ag23;
+Wrn472 = MatrixWithName("rmsn_472.weight", D, 1);
+Wg23 = MatrixWithName("gemm_189.weight", I, D, 1, 1);
+Wu23 = MatrixWithName("gemm_190.weight", I, D, 1, 1);
+Wd23 = MatrixWithName("gemm_191.weight", D, I, 1, 1);
+Wrn473 = MatrixWithName("rmsn_473.weight", D, 1);
+Rf23 = rmsNorm(Rx23, Wrn472, 1e-5);
+Rfm23 = Rf23 + elementMul(Rf23, sf23);
+Fg23 = batchedMul(Wg23, Rfm23);
+Fu23 = batchedMul(Wu23, Rfm23);
+Ff23 = batchedMul(Wd23, elementMul(silu(Fg23), Fu23));
+Fn23 = rmsNorm(Ff23, Wrn473, 1e-5);
+Fgg23 = elementMul(Fn23, gf23);
+X24 = Rx23 + Fgg23;
+Wmod24 = MatrixWithName("gemm_192.weight", 15360, 256, 1, 1);
+bmod24 = MatrixWithName("gemm_192.bias", 15360, 1, 1, 1);
+mod24 = batchedMul(Wmod24, t_emb) + bmod24;
+sa24 = sliceW(mod24, 0, D);
+ga24 = tanh(sliceW(mod24, D, D));
+sf24 = sliceW(mod24, 2*D, D);
+gf24 = tanh(sliceW(mod24, 3*D, D));
+Wrn474 = MatrixWithName("rmsn_474.weight", D, 1);
+Xn24 = rmsNorm(X24, Wrn474, 1e-5);
+Xm24 = Xn24 + elementMul(Xn24, sa24);
+Wq24 = MatrixWithName("gemm_193.weight", D, D, 1, 1);
+Wk24 = MatrixWithName("gemm_194.weight", D, D, 1, 1);
+Wv24 = MatrixWithName("gemm_195.weight", D, D, 1, 1);
+Wrn475 = MatrixWithName("rmsn_475.weight", hd, 1);
+Wrn476 = MatrixWithName("rmsn_476.weight", hd, 1);
+Wo24 = MatrixWithName("gemm_196.weight", D, D, 1, 1);
+Wrn477 = MatrixWithName("rmsn_477.weight", D, 1);
+Qr24 = permute(reshape(batchedMul(Wq24, Xm24), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr24 = permute(reshape(batchedMul(Wk24, Xm24), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr24 = permute(reshape(batchedMul(Wv24, Xm24), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn24 = rmsNorm(Qr24, Wrn475, 1e-5);
+Kn24 = rmsNorm(Kr24, Wrn476, 1e-5);
+At24 = attention(rope2(Qn24, cos1, sin1), rope2(Kn24, cos1, sin1), Vr24, hd, 0);
+Ao24 = batchedMul(Wo24, reshape(permute(At24, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An24 = rmsNorm(Ao24, Wrn477, 1e-5);
+Ag24 = elementMul(An24, ga24);
+Rx24 = X24 + Ag24;
+Wrn478 = MatrixWithName("rmsn_478.weight", D, 1);
+Wg24 = MatrixWithName("gemm_197.weight", I, D, 1, 1);
+Wu24 = MatrixWithName("gemm_198.weight", I, D, 1, 1);
+Wd24 = MatrixWithName("gemm_199.weight", D, I, 1, 1);
+Wrn479 = MatrixWithName("rmsn_479.weight", D, 1);
+Rf24 = rmsNorm(Rx24, Wrn478, 1e-5);
+Rfm24 = Rf24 + elementMul(Rf24, sf24);
+Fg24 = batchedMul(Wg24, Rfm24);
+Fu24 = batchedMul(Wu24, Rfm24);
+Ff24 = batchedMul(Wd24, elementMul(silu(Fg24), Fu24));
+Fn24 = rmsNorm(Ff24, Wrn479, 1e-5);
+Fgg24 = elementMul(Fn24, gf24);
+X25 = Rx24 + Fgg24;
+Wmod25 = MatrixWithName("gemm_200.weight", 15360, 256, 1, 1);
+bmod25 = MatrixWithName("gemm_200.bias", 15360, 1, 1, 1);
+mod25 = batchedMul(Wmod25, t_emb) + bmod25;
+sa25 = sliceW(mod25, 0, D);
+ga25 = tanh(sliceW(mod25, D, D));
+sf25 = sliceW(mod25, 2*D, D);
+gf25 = tanh(sliceW(mod25, 3*D, D));
+Wrn480 = MatrixWithName("rmsn_480.weight", D, 1);
+Xn25 = rmsNorm(X25, Wrn480, 1e-5);
+Xm25 = Xn25 + elementMul(Xn25, sa25);
+Wq25 = MatrixWithName("gemm_201.weight", D, D, 1, 1);
+Wk25 = MatrixWithName("gemm_202.weight", D, D, 1, 1);
+Wv25 = MatrixWithName("gemm_203.weight", D, D, 1, 1);
+Wrn481 = MatrixWithName("rmsn_481.weight", hd, 1);
+Wrn482 = MatrixWithName("rmsn_482.weight", hd, 1);
+Wo25 = MatrixWithName("gemm_204.weight", D, D, 1, 1);
+Wrn483 = MatrixWithName("rmsn_483.weight", D, 1);
+Qr25 = permute(reshape(batchedMul(Wq25, Xm25), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr25 = permute(reshape(batchedMul(Wk25, Xm25), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr25 = permute(reshape(batchedMul(Wv25, Xm25), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn25 = rmsNorm(Qr25, Wrn481, 1e-5);
+Kn25 = rmsNorm(Kr25, Wrn482, 1e-5);
+At25 = attention(rope2(Qn25, cos1, sin1), rope2(Kn25, cos1, sin1), Vr25, hd, 0);
+Ao25 = batchedMul(Wo25, reshape(permute(At25, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An25 = rmsNorm(Ao25, Wrn483, 1e-5);
+Ag25 = elementMul(An25, ga25);
+Rx25 = X25 + Ag25;
+Wrn484 = MatrixWithName("rmsn_484.weight", D, 1);
+Wg25 = MatrixWithName("gemm_205.weight", I, D, 1, 1);
+Wu25 = MatrixWithName("gemm_206.weight", I, D, 1, 1);
+Wd25 = MatrixWithName("gemm_207.weight", D, I, 1, 1);
+Wrn485 = MatrixWithName("rmsn_485.weight", D, 1);
+Rf25 = rmsNorm(Rx25, Wrn484, 1e-5);
+Rfm25 = Rf25 + elementMul(Rf25, sf25);
+Fg25 = batchedMul(Wg25, Rfm25);
+Fu25 = batchedMul(Wu25, Rfm25);
+Ff25 = batchedMul(Wd25, elementMul(silu(Fg25), Fu25));
+Fn25 = rmsNorm(Ff25, Wrn485, 1e-5);
+Fgg25 = elementMul(Fn25, gf25);
+X26 = Rx25 + Fgg25;
+Wmod26 = MatrixWithName("gemm_208.weight", 15360, 256, 1, 1);
+bmod26 = MatrixWithName("gemm_208.bias", 15360, 1, 1, 1);
+mod26 = batchedMul(Wmod26, t_emb) + bmod26;
+sa26 = sliceW(mod26, 0, D);
+ga26 = tanh(sliceW(mod26, D, D));
+sf26 = sliceW(mod26, 2*D, D);
+gf26 = tanh(sliceW(mod26, 3*D, D));
+Wrn486 = MatrixWithName("rmsn_486.weight", D, 1);
+Xn26 = rmsNorm(X26, Wrn486, 1e-5);
+Xm26 = Xn26 + elementMul(Xn26, sa26);
+Wq26 = MatrixWithName("gemm_209.weight", D, D, 1, 1);
+Wk26 = MatrixWithName("gemm_210.weight", D, D, 1, 1);
+Wv26 = MatrixWithName("gemm_211.weight", D, D, 1, 1);
+Wrn487 = MatrixWithName("rmsn_487.weight", hd, 1);
+Wrn488 = MatrixWithName("rmsn_488.weight", hd, 1);
+Wo26 = MatrixWithName("gemm_212.weight", D, D, 1, 1);
+Wrn489 = MatrixWithName("rmsn_489.weight", D, 1);
+Qr26 = permute(reshape(batchedMul(Wq26, Xm26), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr26 = permute(reshape(batchedMul(Wk26, Xm26), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr26 = permute(reshape(batchedMul(Wv26, Xm26), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn26 = rmsNorm(Qr26, Wrn487, 1e-5);
+Kn26 = rmsNorm(Kr26, Wrn488, 1e-5);
+At26 = attention(rope2(Qn26, cos1, sin1), rope2(Kn26, cos1, sin1), Vr26, hd, 0);
+Ao26 = batchedMul(Wo26, reshape(permute(At26, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An26 = rmsNorm(Ao26, Wrn489, 1e-5);
+Ag26 = elementMul(An26, ga26);
+Rx26 = X26 + Ag26;
+Wrn490 = MatrixWithName("rmsn_490.weight", D, 1);
+Wg26 = MatrixWithName("gemm_213.weight", I, D, 1, 1);
+Wu26 = MatrixWithName("gemm_214.weight", I, D, 1, 1);
+Wd26 = MatrixWithName("gemm_215.weight", D, I, 1, 1);
+Wrn491 = MatrixWithName("rmsn_491.weight", D, 1);
+Rf26 = rmsNorm(Rx26, Wrn490, 1e-5);
+Rfm26 = Rf26 + elementMul(Rf26, sf26);
+Fg26 = batchedMul(Wg26, Rfm26);
+Fu26 = batchedMul(Wu26, Rfm26);
+Ff26 = batchedMul(Wd26, elementMul(silu(Fg26), Fu26));
+Fn26 = rmsNorm(Ff26, Wrn491, 1e-5);
+Fgg26 = elementMul(Fn26, gf26);
+X27 = Rx26 + Fgg26;
+Wmod27 = MatrixWithName("gemm_216.weight", 15360, 256, 1, 1);
+bmod27 = MatrixWithName("gemm_216.bias", 15360, 1, 1, 1);
+mod27 = batchedMul(Wmod27, t_emb) + bmod27;
+sa27 = sliceW(mod27, 0, D);
+ga27 = tanh(sliceW(mod27, D, D));
+sf27 = sliceW(mod27, 2*D, D);
+gf27 = tanh(sliceW(mod27, 3*D, D));
+Wrn492 = MatrixWithName("rmsn_492.weight", D, 1);
+Xn27 = rmsNorm(X27, Wrn492, 1e-5);
+Xm27 = Xn27 + elementMul(Xn27, sa27);
+Wq27 = MatrixWithName("gemm_217.weight", D, D, 1, 1);
+Wk27 = MatrixWithName("gemm_218.weight", D, D, 1, 1);
+Wv27 = MatrixWithName("gemm_219.weight", D, D, 1, 1);
+Wrn493 = MatrixWithName("rmsn_493.weight", hd, 1);
+Wrn494 = MatrixWithName("rmsn_494.weight", hd, 1);
+Wo27 = MatrixWithName("gemm_220.weight", D, D, 1, 1);
+Wrn495 = MatrixWithName("rmsn_495.weight", D, 1);
+Qr27 = permute(reshape(batchedMul(Wq27, Xm27), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr27 = permute(reshape(batchedMul(Wk27, Xm27), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr27 = permute(reshape(batchedMul(Wv27, Xm27), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn27 = rmsNorm(Qr27, Wrn493, 1e-5);
+Kn27 = rmsNorm(Kr27, Wrn494, 1e-5);
+At27 = attention(rope2(Qn27, cos1, sin1), rope2(Kn27, cos1, sin1), Vr27, hd, 0);
+Ao27 = batchedMul(Wo27, reshape(permute(At27, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An27 = rmsNorm(Ao27, Wrn495, 1e-5);
+Ag27 = elementMul(An27, ga27);
+Rx27 = X27 + Ag27;
+Wrn496 = MatrixWithName("rmsn_496.weight", D, 1);
+Wg27 = MatrixWithName("gemm_221.weight", I, D, 1, 1);
+Wu27 = MatrixWithName("gemm_222.weight", I, D, 1, 1);
+Wd27 = MatrixWithName("gemm_223.weight", D, I, 1, 1);
+Wrn497 = MatrixWithName("rmsn_497.weight", D, 1);
+Rf27 = rmsNorm(Rx27, Wrn496, 1e-5);
+Rfm27 = Rf27 + elementMul(Rf27, sf27);
+Fg27 = batchedMul(Wg27, Rfm27);
+Fu27 = batchedMul(Wu27, Rfm27);
+Ff27 = batchedMul(Wd27, elementMul(silu(Fg27), Fu27));
+Fn27 = rmsNorm(Ff27, Wrn497, 1e-5);
+Fgg27 = elementMul(Fn27, gf27);
+X28 = Rx27 + Fgg27;
+Wmod28 = MatrixWithName("gemm_224.weight", 15360, 256, 1, 1);
+bmod28 = MatrixWithName("gemm_224.bias", 15360, 1, 1, 1);
+mod28 = batchedMul(Wmod28, t_emb) + bmod28;
+sa28 = sliceW(mod28, 0, D);
+ga28 = tanh(sliceW(mod28, D, D));
+sf28 = sliceW(mod28, 2*D, D);
+gf28 = tanh(sliceW(mod28, 3*D, D));
+Wrn498 = MatrixWithName("rmsn_498.weight", D, 1);
+Xn28 = rmsNorm(X28, Wrn498, 1e-5);
+Xm28 = Xn28 + elementMul(Xn28, sa28);
+Wq28 = MatrixWithName("gemm_225.weight", D, D, 1, 1);
+Wk28 = MatrixWithName("gemm_226.weight", D, D, 1, 1);
+Wv28 = MatrixWithName("gemm_227.weight", D, D, 1, 1);
+Wrn499 = MatrixWithName("rmsn_499.weight", hd, 1);
+Wrn500 = MatrixWithName("rmsn_500.weight", hd, 1);
+Wo28 = MatrixWithName("gemm_228.weight", D, D, 1, 1);
+Wrn501 = MatrixWithName("rmsn_501.weight", D, 1);
+Qr28 = permute(reshape(batchedMul(Wq28, Xm28), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr28 = permute(reshape(batchedMul(Wk28, Xm28), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr28 = permute(reshape(batchedMul(Wv28, Xm28), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn28 = rmsNorm(Qr28, Wrn499, 1e-5);
+Kn28 = rmsNorm(Kr28, Wrn500, 1e-5);
+At28 = attention(rope2(Qn28, cos1, sin1), rope2(Kn28, cos1, sin1), Vr28, hd, 0);
+Ao28 = batchedMul(Wo28, reshape(permute(At28, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An28 = rmsNorm(Ao28, Wrn501, 1e-5);
+Ag28 = elementMul(An28, ga28);
+Rx28 = X28 + Ag28;
+Wrn502 = MatrixWithName("rmsn_502.weight", D, 1);
+Wg28 = MatrixWithName("gemm_229.weight", I, D, 1, 1);
+Wu28 = MatrixWithName("gemm_230.weight", I, D, 1, 1);
+Wd28 = MatrixWithName("gemm_231.weight", D, I, 1, 1);
+Wrn503 = MatrixWithName("rmsn_503.weight", D, 1);
+Rf28 = rmsNorm(Rx28, Wrn502, 1e-5);
+Rfm28 = Rf28 + elementMul(Rf28, sf28);
+Fg28 = batchedMul(Wg28, Rfm28);
+Fu28 = batchedMul(Wu28, Rfm28);
+Ff28 = batchedMul(Wd28, elementMul(silu(Fg28), Fu28));
+Fn28 = rmsNorm(Ff28, Wrn503, 1e-5);
+Fgg28 = elementMul(Fn28, gf28);
+X29 = Rx28 + Fgg28;
+Wmod29 = MatrixWithName("gemm_232.weight", 15360, 256, 1, 1);
+bmod29 = MatrixWithName("gemm_232.bias", 15360, 1, 1, 1);
+mod29 = batchedMul(Wmod29, t_emb) + bmod29;
+sa29 = sliceW(mod29, 0, D);
+ga29 = tanh(sliceW(mod29, D, D));
+sf29 = sliceW(mod29, 2*D, D);
+gf29 = tanh(sliceW(mod29, 3*D, D));
+Wrn504 = MatrixWithName("rmsn_504.weight", D, 1);
+Xn29 = rmsNorm(X29, Wrn504, 1e-5);
+Xm29 = Xn29 + elementMul(Xn29, sa29);
+Wq29 = MatrixWithName("gemm_233.weight", D, D, 1, 1);
+Wk29 = MatrixWithName("gemm_234.weight", D, D, 1, 1);
+Wv29 = MatrixWithName("gemm_235.weight", D, D, 1, 1);
+Wrn505 = MatrixWithName("rmsn_505.weight", hd, 1);
+Wrn506 = MatrixWithName("rmsn_506.weight", hd, 1);
+Wo29 = MatrixWithName("gemm_236.weight", D, D, 1, 1);
+Wrn507 = MatrixWithName("rmsn_507.weight", D, 1);
+Qr29 = permute(reshape(batchedMul(Wq29, Xm29), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Kr29 = permute(reshape(batchedMul(Wk29, Xm29), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Vr29 = permute(reshape(batchedMul(Wv29, Xm29), {hd, HB, SEQ, 1}), {0, 2, 3, 1});
+Qn29 = rmsNorm(Qr29, Wrn505, 1e-5);
+Kn29 = rmsNorm(Kr29, Wrn506, 1e-5);
+At29 = attention(rope2(Qn29, cos1, sin1), rope2(Kn29, cos1, sin1), Vr29, hd, 0);
+Ao29 = batchedMul(Wo29, reshape(permute(At29, {0, 3, 1, 2}), {D, SEQ, 1, 1}));
+An29 = rmsNorm(Ao29, Wrn507, 1e-5);
+Ag29 = elementMul(An29, ga29);
+Rx29 = X29 + Ag29;
+Wrn508 = MatrixWithName("rmsn_508.weight", D, 1);
+Wg29 = MatrixWithName("gemm_237.weight", I, D, 1, 1);
+Wu29 = MatrixWithName("gemm_238.weight", I, D, 1, 1);
+Wd29 = MatrixWithName("gemm_239.weight", D, I, 1, 1);
+Wrn509 = MatrixWithName("rmsn_509.weight", D, 1);
+Rf29 = rmsNorm(Rx29, Wrn508, 1e-5);
+Rfm29 = Rf29 + elementMul(Rf29, sf29);
+Fg29 = batchedMul(Wg29, Rfm29);
+Fu29 = batchedMul(Wu29, Rfm29);
+Ff29 = batchedMul(Wd29, elementMul(silu(Fg29), Fu29));
+Fn29 = rmsNorm(Ff29, Wrn509, 1e-5);
+Fgg29 = elementMul(Fn29, gf29);
+X30 = Rx29 + Fgg29;
+
+setXY(Xin, X30);
+'
+

net_vae.ini

@@ -0,0 +1,219 @@
+[train]
+load_net    = 1
+load_file=z_image_turbo_vae.cccc.bin
+train_epochs = 0
+Batch       = 1
+gpu         = 1
+data_type   = float
+output_log  = 0
+need_free_mem = 0
+disable_activation_memory_reuse = 1
+
+[net]
+structure='
+B=1;
+Xinp = Matrix(lH, lW, 16, B);
+Wc32=MatrixWithName("conv_32.weight",3,3,16,512); bc32=MatrixWithName("conv_32.bias",1,1,512,1);
+Z0=conv(Xinp,Wc32,{1,1},{1,1})+bc32;
+print_message(Z0);
+Gs67=MatrixWithName("gn_67.weight",512,1);
+Gb67=MatrixWithName("gn_67.bias",512,1);
+Wc33=MatrixWithName("conv_33.weight",3,3,512,512);
+bc33=MatrixWithName("conv_33.bias",1,1,512,1);
+Gs68=MatrixWithName("gn_68.weight",512,1);
+Gb68=MatrixWithName("gn_68.bias",512,1);
+Wc34=MatrixWithName("conv_34.weight",3,3,512,512);
+bc34=MatrixWithName("conv_34.bias",1,1,512,1);
+Zr33a=conv(silu(groupNorm(Z0,32,Gs67,Gb67,1e-6)),Wc33,{1,1},{1,1})+bc33;
+Zr33b=conv(silu(groupNorm(Zr33a,32,Gs68,Gb68,1e-6)),Wc34,{1,1},{1,1})+bc34;
+Z1=Z0+Zr33b;
+print_message(Z1);
+Gs69=MatrixWithName("gn_69.weight",512,1);
+Gb69=MatrixWithName("gn_69.bias",512,1);
+Za_n=groupNorm(Z1,32,Gs69,Gb69,1e-6);
+Za_p=permute(Za_n,{2,0,1,3});
+Za_r=reshapeBatch(Za_p,{512,lH*lW,1,1});
+Wq97=MatrixWithName("attention_97.q_weight",512,512,1,1);
+bq97=MatrixWithName("attention_97.q_bias",512,1,1,1);
+Wk97=MatrixWithName("attention_97.k_weight",512,512,1,1);
+bk97=MatrixWithName("attention_97.k_bias",512,1,1,1);
+Wv97=MatrixWithName("attention_97.v_weight",512,512,1,1);
+bv97=MatrixWithName("attention_97.v_bias",512,1,1,1);
+Wo97=MatrixWithName("attention_97.out_weight",512,512,1,1);
+bo97=MatrixWithName("attention_97.out_bias",512,1,1,1);
+Q97=batchedMul(Wq97,Za_r,1,0)+bq97;
+K97=batchedMul(Wk97,Za_r,1,0)+bk97;
+V97=batchedMul(Wv97,Za_r,1,0)+bv97;
+A97=attention(Q97,K97,V97,512,0);
+Ao97=batchedMul(Wo97,A97,1,0)+bo97;
+Ao97s=reshapeBatch(Ao97,{512,lH,lW,1});
+Ao97b=permute(Ao97s,{1,2,0,3});
+Z1a=Z1+Ao97b;
+print_message(Z1a);
+Gs70=MatrixWithName("gn_70.weight",512,1);
+Gb70=MatrixWithName("gn_70.bias",512,1);
+Wc35=MatrixWithName("conv_35.weight",3,3,512,512);
+bc35=MatrixWithName("conv_35.bias",1,1,512,1);
+Gs71=MatrixWithName("gn_71.weight",512,1);
+Gb71=MatrixWithName("gn_71.bias",512,1);
+Wc36=MatrixWithName("conv_36.weight",3,3,512,512);
+bc36=MatrixWithName("conv_36.bias",1,1,512,1);
+Zr35a=conv(silu(groupNorm(Z1a,32,Gs70,Gb70,1e-6)),Wc35,{1,1},{1,1})+bc35;
+Zr35b=conv(silu(groupNorm(Zr35a,32,Gs71,Gb71,1e-6)),Wc36,{1,1},{1,1})+bc36;
+Z2=Z1a+Zr35b;
+Gs72=MatrixWithName("gn_72.weight",512,1);
+Gb72=MatrixWithName("gn_72.bias",512,1);
+Wc37=MatrixWithName("conv_37.weight",3,3,512,512);
+bc37=MatrixWithName("conv_37.bias",1,1,512,1);
+Gs73=MatrixWithName("gn_73.weight",512,1);
+Gb73=MatrixWithName("gn_73.bias",512,1);
+Wc38=MatrixWithName("conv_38.weight",3,3,512,512);
+bc38=MatrixWithName("conv_38.bias",1,1,512,1);
+Zr37a=conv(silu(groupNorm(Z2,32,Gs72,Gb72,1e-6)),Wc37,{1,1},{1,1})+bc37;
+Zr37b=conv(silu(groupNorm(Zr37a,32,Gs73,Gb73,1e-6)),Wc38,{1,1},{1,1})+bc38;
+Z3=Z2+Zr37b;
+Gs74=MatrixWithName("gn_74.weight",512,1);
+Gb74=MatrixWithName("gn_74.bias",512,1);
+Wc39=MatrixWithName("conv_39.weight",3,3,512,512);
+bc39=MatrixWithName("conv_39.bias",1,1,512,1);
+Gs75=MatrixWithName("gn_75.weight",512,1);
+Gb75=MatrixWithName("gn_75.bias",512,1);
+Wc40=MatrixWithName("conv_40.weight",3,3,512,512);
+bc40=MatrixWithName("conv_40.bias",1,1,512,1);
+Zr39a=conv(silu(groupNorm(Z3,32,Gs74,Gb74,1e-6)),Wc39,{1,1},{1,1})+bc39;
+Zr39b=conv(silu(groupNorm(Zr39a,32,Gs75,Gb75,1e-6)),Wc40,{1,1},{1,1})+bc40;
+Z4=Z3+Zr39b;
+Gs76=MatrixWithName("gn_76.weight",512,1);
+Gb76=MatrixWithName("gn_76.bias",512,1);
+Wc41=MatrixWithName("conv_41.weight",3,3,512,512);
+bc41=MatrixWithName("conv_41.bias",1,1,512,1);
+Gs77=MatrixWithName("gn_77.weight",512,1);
+Gb77=MatrixWithName("gn_77.bias",512,1);
+Wc42=MatrixWithName("conv_42.weight",3,3,512,512);
+bc42=MatrixWithName("conv_42.bias",1,1,512,1);
+Zr41a=conv(silu(groupNorm(Z4,32,Gs76,Gb76,1e-6)),Wc41,{1,1},{1,1})+bc41;
+Zr41b=conv(silu(groupNorm(Zr41a,32,Gs77,Gb77,1e-6)),Wc42,{1,1},{1,1})+bc42;
+Z5=Z4+Zr41b;
+Z5u=upsample(Z5,2,2,0);
+Wc43=MatrixWithName("conv_43.weight",3,3,512,512); bc43=MatrixWithName("conv_43.bias",1,1,512,1);
+Z6=conv(Z5u,Wc43,{1,1},{1,1})+bc43;
+Gs78=MatrixWithName("gn_78.weight",512,1);
+Gb78=MatrixWithName("gn_78.bias",512,1);
+Wc44=MatrixWithName("conv_44.weight",3,3,512,512);
+bc44=MatrixWithName("conv_44.bias",1,1,512,1);
+Gs79=MatrixWithName("gn_79.weight",512,1);
+Gb79=MatrixWithName("gn_79.bias",512,1);
+Wc45=MatrixWithName("conv_45.weight",3,3,512,512);
+bc45=MatrixWithName("conv_45.bias",1,1,512,1);
+Zr44a=conv(silu(groupNorm(Z6,32,Gs78,Gb78,1e-6)),Wc44,{1,1},{1,1})+bc44;
+Zr44b=conv(silu(groupNorm(Zr44a,32,Gs79,Gb79,1e-6)),Wc45,{1,1},{1,1})+bc45;
+Z7=Z6+Zr44b;
+Gs80=MatrixWithName("gn_80.weight",512,1);
+Gb80=MatrixWithName("gn_80.bias",512,1);
+Wc46=MatrixWithName("conv_46.weight",3,3,512,512);
+bc46=MatrixWithName("conv_46.bias",1,1,512,1);
+Gs81=MatrixWithName("gn_81.weight",512,1);
+Gb81=MatrixWithName("gn_81.bias",512,1);
+Wc47=MatrixWithName("conv_47.weight",3,3,512,512);
+bc47=MatrixWithName("conv_47.bias",1,1,512,1);
+Zr46a=conv(silu(groupNorm(Z7,32,Gs80,Gb80,1e-6)),Wc46,{1,1},{1,1})+bc46;
+Zr46b=conv(silu(groupNorm(Zr46a,32,Gs81,Gb81,1e-6)),Wc47,{1,1},{1,1})+bc47;
+Z8=Z7+Zr46b;
+Gs82=MatrixWithName("gn_82.weight",512,1);
+Gb82=MatrixWithName("gn_82.bias",512,1);
+Wc48=MatrixWithName("conv_48.weight",3,3,512,512);
+bc48=MatrixWithName("conv_48.bias",1,1,512,1);
+Gs83=MatrixWithName("gn_83.weight",512,1);
+Gb83=MatrixWithName("gn_83.bias",512,1);
+Wc49=MatrixWithName("conv_49.weight",3,3,512,512);
+bc49=MatrixWithName("conv_49.bias",1,1,512,1);
+Zr48a=conv(silu(groupNorm(Z8,32,Gs82,Gb82,1e-6)),Wc48,{1,1},{1,1})+bc48;
+Zr48b=conv(silu(groupNorm(Zr48a,32,Gs83,Gb83,1e-6)),Wc49,{1,1},{1,1})+bc49;
+Z9=Z8+Zr48b;
+Z9u=upsample(Z9,2,2,0);
+Wc50=MatrixWithName("conv_50.weight",3,3,512,512); bc50=MatrixWithName("conv_50.bias",1,1,512,1);
+Z10=conv(Z9u,Wc50,{1,1},{1,1})+bc50;
+Gs84=MatrixWithName("gn_84.weight",512,1);
+Gb84=MatrixWithName("gn_84.bias",512,1);
+Wc51=MatrixWithName("conv_51.weight",3,3,512,256);
+bc51=MatrixWithName("conv_51.bias",1,1,256,1);
+Gs85=MatrixWithName("gn_85.weight",256,1);
+Gb85=MatrixWithName("gn_85.bias",256,1);
+Wc52=MatrixWithName("conv_52.weight",3,3,256,256);
+bc52=MatrixWithName("conv_52.bias",1,1,256,1);
+Wc53=MatrixWithName("conv_53.weight",1,1,512,256);
+bc53=MatrixWithName("conv_53.bias",1,1,256,1);
+Zr51a=conv(silu(groupNorm(Z10,32,Gs84,Gb84,1e-6)),Wc51,{1,1},{1,1})+bc51;
+Zr51b=conv(silu(groupNorm(Zr51a,32,Gs85,Gb85,1e-6)),Wc52,{1,1},{1,1})+bc52;
+Zsk53=conv(Z10,Wc53,{1,1},{0,0})+bc53;
+Z11=Zsk53+Zr51b;
+Gs86=MatrixWithName("gn_86.weight",256,1);
+Gb86=MatrixWithName("gn_86.bias",256,1);
+Wc54=MatrixWithName("conv_54.weight",3,3,256,256);
+bc54=MatrixWithName("conv_54.bias",1,1,256,1);
+Gs87=MatrixWithName("gn_87.weight",256,1);
+Gb87=MatrixWithName("gn_87.bias",256,1);
+Wc55=MatrixWithName("conv_55.weight",3,3,256,256);
+bc55=MatrixWithName("conv_55.bias",1,1,256,1);
+Zr54a=conv(silu(groupNorm(Z11,32,Gs86,Gb86,1e-6)),Wc54,{1,1},{1,1})+bc54;
+Zr54b=conv(silu(groupNorm(Zr54a,32,Gs87,Gb87,1e-6)),Wc55,{1,1},{1,1})+bc55;
+Z12=Z11+Zr54b;
+Gs88=MatrixWithName("gn_88.weight",256,1);
+Gb88=MatrixWithName("gn_88.bias",256,1);
+Wc56=MatrixWithName("conv_56.weight",3,3,256,256);
+bc56=MatrixWithName("conv_56.bias",1,1,256,1);
+Gs89=MatrixWithName("gn_89.weight",256,1);
+Gb89=MatrixWithName("gn_89.bias",256,1);
+Wc57=MatrixWithName("conv_57.weight",3,3,256,256);
+bc57=MatrixWithName("conv_57.bias",1,1,256,1);
+Zr56a=conv(silu(groupNorm(Z12,32,Gs88,Gb88,1e-6)),Wc56,{1,1},{1,1})+bc56;
+Zr56b=conv(silu(groupNorm(Zr56a,32,Gs89,Gb89,1e-6)),Wc57,{1,1},{1,1})+bc57;
+Z13=Z12+Zr56b;
+Z13u=upsample(Z13,2,2,0);
+Wc58=MatrixWithName("conv_58.weight",3,3,256,256); bc58=MatrixWithName("conv_58.bias",1,1,256,1);
+Z14=conv(Z13u,Wc58,{1,1},{1,1})+bc58;
+Gs90=MatrixWithName("gn_90.weight",256,1);
+Gb90=MatrixWithName("gn_90.bias",256,1);
+Wc59=MatrixWithName("conv_59.weight",3,3,256,128);
+bc59=MatrixWithName("conv_59.bias",1,1,128,1);
+Gs91=MatrixWithName("gn_91.weight",128,1);
+Gb91=MatrixWithName("gn_91.bias",128,1);
+Wc60=MatrixWithName("conv_60.weight",3,3,128,128);
+bc60=MatrixWithName("conv_60.bias",1,1,128,1);
+Wc61=MatrixWithName("conv_61.weight",1,1,256,128);
+bc61=MatrixWithName("conv_61.bias",1,1,128,1);
+Zr59a=conv(silu(groupNorm(Z14,32,Gs90,Gb90,1e-6)),Wc59,{1,1},{1,1})+bc59;
+Zr59b=conv(silu(groupNorm(Zr59a,32,Gs91,Gb91,1e-6)),Wc60,{1,1},{1,1})+bc60;
+Zsk61=conv(Z14,Wc61,{1,1},{0,0})+bc61;
+Z15=Zsk61+Zr59b;
+Gs92=MatrixWithName("gn_92.weight",128,1);
+Gb92=MatrixWithName("gn_92.bias",128,1);
+Wc62=MatrixWithName("conv_62.weight",3,3,128,128);
+bc62=MatrixWithName("conv_62.bias",1,1,128,1);
+Gs93=MatrixWithName("gn_93.weight",128,1);
+Gb93=MatrixWithName("gn_93.bias",128,1);
+Wc63=MatrixWithName("conv_63.weight",3,3,128,128);
+bc63=MatrixWithName("conv_63.bias",1,1,128,1);
+Zr62a=conv(silu(groupNorm(Z15,32,Gs92,Gb92,1e-6)),Wc62,{1,1},{1,1})+bc62;
+Zr62b=conv(silu(groupNorm(Zr62a,32,Gs93,Gb93,1e-6)),Wc63,{1,1},{1,1})+bc63;
+Z16=Z15+Zr62b;
+Gs94=MatrixWithName("gn_94.weight",128,1);
+Gb94=MatrixWithName("gn_94.bias",128,1);
+Wc64=MatrixWithName("conv_64.weight",3,3,128,128);
+bc64=MatrixWithName("conv_64.bias",1,1,128,1);
+Gs95=MatrixWithName("gn_95.weight",128,1);
+Gb95=MatrixWithName("gn_95.bias",128,1);
+Wc65=MatrixWithName("conv_65.weight",3,3,128,128);
+bc65=MatrixWithName("conv_65.bias",1,1,128,1);
+Zr64a=conv(silu(groupNorm(Z16,32,Gs94,Gb94,1e-6)),Wc64,{1,1},{1,1})+bc64;
+Zr64b=conv(silu(groupNorm(Zr64a,32,Gs95,Gb95,1e-6)),Wc65,{1,1},{1,1})+bc65;
+Z17=Z16+Zr64b;
+Gs96=MatrixWithName("gn_96.weight",128,1);
+Gb96=MatrixWithName("gn_96.bias",128,1);
+Wc66=MatrixWithName("conv_66.weight",3,3,128,3);
+bc66=MatrixWithName("conv_66.bias",1,1,3,1);
+Y=conv(silu(groupNorm(Z17,32,Gs96,Gb96,1e-6)),Wc66,{1,1},{1,1})+bc66;
+print_message(Y);
+setXY(Xinp, Y);
+'
+

net_xemb.ini

@@ -0,0 +1,20 @@
+[train]
+load_net    = 1
+load_file=z_image_turbo_transformer_all_x_embedder.cccc.bin
+train_epochs = 0
+Batch       = 1
+gpu         = 1
+data_type = half
+output_log  = 0
+need_free_mem = 0
+
+[net]
+structure='
+B=1;
+W = MatrixWithName("gemm_0.weight", 3840, 64, 1, 1);
+b = MatrixWithName("gemm_0.bias", 3840, 1, 1, 1);
+X = Matrix(64, P, 1, B);
+Y = batchedMul(W, X) + b;
+setXY(X, Y);
+'
+

z_image_turbo_text_encoder.cccc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3db3dc860798d8e0978aa1a1a318acbffc1c0bc3a8e659b769186ca3bed8a53
+size 7843085329

z_image_turbo_transformer_all_final_layer.cccc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d315e812a77b3523d9b9e6f5d4d09e233c49acf408884506b2d5558b095e326
+size 2465674

z_image_turbo_transformer_all_x_embedder.cccc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1788f6bd48798a99a5c0b330b80c64903b62bd17ec31dbc20c58914bf2a6a8d
+size 499396

z_image_turbo_transformer_cap_embedder.cccc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0c4d9ab33a4a11579356b7de35fc3e94394db349d26e95afaacf463402ecef6
+size 19673829

z_image_turbo_transformer_context_refiner.cccc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f2df00443b15c29229a841c6bc394d8e054558163ed8bd99ae4715267366ab4
+size 707852386

z_image_turbo_transformer_noise_refiner.cccc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f1e6b0a368167fc4c3001959b7ddc86b11d8f178f777d839bdd49b4a7541fb6
+size 723642613

z_image_turbo_transformer_t_embedder.cccc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06d39ed60ad841417a175a019f1a7f5e4e50723be36c3c34371dc94678d4879f
+size 1051688

z_image_turbo_transformer_unified.cccc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:371b79b1491c8c2c019c9371432c29defe0e50c3240b134dde6ed7271a226770
+size 10854638523

z_image_turbo_vae.cccc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3a36f49e125563517392e4ebc20a1309e818b8e69107097ed2bb57f77f179af
+size 99095867